require(data.table)
## Loading required package: data.table
require(ggplot2)
## Loading required package: ggplot2
require(repr)
## Loading required package: repr
require(rpart)
## Loading required package: rpart
require(rattle)
## Loading required package: rattle
## Loading required package: tibble
## Loading required package: bitops
## Rattle: A free graphical interface for data science with R.
## Version 5.4.0 Copyright (c) 2006-2020 Togaware Pty Ltd.
## Type 'rattle()' to shake, rattle, and roll your data.
require(TSrepr)
## Loading required package: TSrepr
require(TSdist)
## Loading required package: TSdist
## Loading required package: proxy
## 
## Attaching package: 'proxy'
## The following objects are masked from 'package:stats':
## 
##     as.dist, dist
## The following object is masked from 'package:base':
## 
##     as.matrix
## Error: package or namespace load failed for 'TSdist' in loadNamespace(i, c(lib.loc, .libPaths()), versionCheck = vI[[i]]):
##  there is no package called 'rgl'
require(dtw)
## Loading required package: dtw
## Loaded dtw v1.22-3. See ?dtw for help, citation("dtw") for use in publication.
require(zoo)
## Loading required package: zoo
## 
## Attaching package: 'zoo'
## The following objects are masked from 'package:base':
## 
##     as.Date, as.Date.numeric
require(genlasso)
## Loading required package: genlasso
## Loading required package: Matrix
## 
## Attaching package: 'Matrix'
## The following object is masked from 'package:bitops':
## 
##     %&%
require(e1071)
## Loading required package: e1071
require(lubridate)
## Loading required package: lubridate
## 
## Attaching package: 'lubridate'
## The following objects are masked from 'package:data.table':
## 
##     hour, isoweek, mday, minute, month, quarter, second, wday, week,
##     yday, year
## The following objects are masked from 'package:base':
## 
##     date, intersect, setdiff, union
require(forecast)
## Loading required package: forecast
## Registered S3 method overwritten by 'quantmod':
##   method            from
##   as.zoo.data.frame zoo
require(dplyr)
## Loading required package: dplyr
## 
## Attaching package: 'dplyr'
## The following objects are masked from 'package:data.table':
## 
##     between, first, last
## The following objects are masked from 'package:stats':
## 
##     filter, lag
## The following objects are masked from 'package:base':
## 
##     intersect, setdiff, setequal, union
options(repr.plot.width=15, repr.plot.height=8)

current_folder=getwd()

The latest input

dat=fread('bulk_imbalance.csv')
weat=fread('2022-01-21_weather.csv')
head(dat)
##          date hour       net upRegulationZeroCoded upRegulationOneCoded
## 1: 2019-01-01    0 -1934.795                     0                    0
## 2: 2019-01-01    1  -750.866                     0                    0
## 3: 2019-01-01    2  -982.767                     0                    0
## 4: 2019-01-01    3  -764.620                     0                    0
## 5: 2019-01-01    4 -1233.289                     0                    0
## 6: 2019-01-01    5 -1469.061                     0                    0
##    upRegulationTwoCoded downRegulationZeroCoded downRegulationOneCoded
## 1:                    0                2288.567                      0
## 2:                    0                 808.083                      0
## 3:                    0                1060.750                      0
## 4:                    0                1120.483                      0
## 5:                    0                1874.633                     56
## 6:                    0                2125.000                     56
##    downRegulationTwoCoded upRegulationDelivered downRegulationDelivered
## 1:                      0                     0               -1934.795
## 2:                      0                     0                -750.866
## 3:                      0                     0                -982.767
## 4:                      0                     0                -764.620
## 5:                      0                     0               -1233.289
## 6:                      0                     0               -1469.061
##    system_direction
## 1:         Negative
## 2:         Negative
## 3:         Negative
## 4:         Negative
## 5:         Negative
## 6:         Negative
head(weat)
##          date hour  lat  lon      variable value
## 1: 2018-12-31    0 36.5 32.5 DSWRF_surface     0
## 2: 2018-12-31    1 36.5 32.5 DSWRF_surface     0
## 3: 2018-12-31    2 36.5 32.5 DSWRF_surface     0
## 4: 2018-12-31    3 36.5 32.5 DSWRF_surface     0
## 5: 2018-12-31    4 36.5 32.5 DSWRF_surface     0
## 6: 2018-12-31    5 36.5 32.5 DSWRF_surface     0
c1 = dat
c1 = c1[,list(date,hour,net,system_direction)]
c1[,datetime:=ymd(date)+dhours(hour)]
c1=c1[order(datetime)]

c2 = c1[,list(datetime,net,system_direction)]
c2
##                   datetime       net system_direction
##     1: 2019-01-01 00:00:00 -1934.795         Negative
##     2: 2019-01-01 01:00:00  -750.866         Negative
##     3: 2019-01-01 02:00:00  -982.767         Negative
##     4: 2019-01-01 03:00:00  -764.620         Negative
##     5: 2019-01-01 04:00:00 -1233.289         Negative
##    ---                                               
## 26804: 2022-01-21 19:00:00     0.000                 
## 26805: 2022-01-21 20:00:00     0.000                 
## 26806: 2022-01-21 21:00:00     0.000                 
## 26807: 2022-01-21 22:00:00     0.000                 
## 26808: 2022-01-21 23:00:00     0.000
w1 = weat
w1[,datetime:=ymd(date)+dhours(hour)]
w1=w1[order(datetime)]
w1
##                date hour   lat   lon      variable value            datetime
##       1: 2018-12-31    0 36.50 32.50 DSWRF_surface     0 2018-12-31 00:00:00
##       2: 2018-12-31    0 37.00 35.50 DSWRF_surface     0 2018-12-31 00:00:00
##       3: 2018-12-31    0 38.00 32.50 DSWRF_surface     0 2018-12-31 00:00:00
##       4: 2018-12-31    0 38.50 27.00 DSWRF_surface     0 2018-12-31 00:00:00
##       5: 2018-12-31    0 39.75 30.50 DSWRF_surface     0 2018-12-31 00:00:00
##      ---                                                                    
## 1129964: 2022-01-24   23 38.00 32.50      wdir_10m    NA 2022-01-24 23:00:00
## 1129965: 2022-01-24   23 38.50 27.00      wdir_10m    NA 2022-01-24 23:00:00
## 1129966: 2022-01-24   23 39.75 30.50      wdir_10m    NA 2022-01-24 23:00:00
## 1129967: 2022-01-24   23 40.00 33.00      wdir_10m    NA 2022-01-24 23:00:00
## 1129968: 2022-01-24   23 41.00 28.75      wdir_10m    NA 2022-01-24 23:00:00

Ordering by date-time for all locations and their weather conditions
Also bringing all parameters together

w1 = w1 %>% 
  mutate(loc = 
           case_when(lat <= 36.50 ~ "loc1" ,
                     lat==37.00 ~ "loc2" ,
                     lat==38.00 ~ "loc3" ,
                     lat==38.50 ~ "loc4" ,
                     lat==39.75 ~ "loc5" ,
                     lat==40.00 ~ "loc6" ,
                     lat>=41.00 ~ "loc7" )
         )
w1= as.data.table(w1)
w1
##                date hour   lat   lon      variable value            datetime
##       1: 2018-12-31    0 36.50 32.50 DSWRF_surface     0 2018-12-31 00:00:00
##       2: 2018-12-31    0 37.00 35.50 DSWRF_surface     0 2018-12-31 00:00:00
##       3: 2018-12-31    0 38.00 32.50 DSWRF_surface     0 2018-12-31 00:00:00
##       4: 2018-12-31    0 38.50 27.00 DSWRF_surface     0 2018-12-31 00:00:00
##       5: 2018-12-31    0 39.75 30.50 DSWRF_surface     0 2018-12-31 00:00:00
##      ---                                                                    
## 1129964: 2022-01-24   23 38.00 32.50      wdir_10m    NA 2022-01-24 23:00:00
## 1129965: 2022-01-24   23 38.50 27.00      wdir_10m    NA 2022-01-24 23:00:00
## 1129966: 2022-01-24   23 39.75 30.50      wdir_10m    NA 2022-01-24 23:00:00
## 1129967: 2022-01-24   23 40.00 33.00      wdir_10m    NA 2022-01-24 23:00:00
## 1129968: 2022-01-24   23 41.00 28.75      wdir_10m    NA 2022-01-24 23:00:00
##           loc
##       1: loc1
##       2: loc2
##       3: loc3
##       4: loc4
##       5: loc5
##      ---     
## 1129964: loc3
## 1129965: loc4
## 1129966: loc5
## 1129967: loc6
## 1129968: loc7
w1[,class:=paste(loc,variable)]
w1
##                date hour   lat   lon      variable value            datetime
##       1: 2018-12-31    0 36.50 32.50 DSWRF_surface     0 2018-12-31 00:00:00
##       2: 2018-12-31    0 37.00 35.50 DSWRF_surface     0 2018-12-31 00:00:00
##       3: 2018-12-31    0 38.00 32.50 DSWRF_surface     0 2018-12-31 00:00:00
##       4: 2018-12-31    0 38.50 27.00 DSWRF_surface     0 2018-12-31 00:00:00
##       5: 2018-12-31    0 39.75 30.50 DSWRF_surface     0 2018-12-31 00:00:00
##      ---                                                                    
## 1129964: 2022-01-24   23 38.00 32.50      wdir_10m    NA 2022-01-24 23:00:00
## 1129965: 2022-01-24   23 38.50 27.00      wdir_10m    NA 2022-01-24 23:00:00
## 1129966: 2022-01-24   23 39.75 30.50      wdir_10m    NA 2022-01-24 23:00:00
## 1129967: 2022-01-24   23 40.00 33.00      wdir_10m    NA 2022-01-24 23:00:00
## 1129968: 2022-01-24   23 41.00 28.75      wdir_10m    NA 2022-01-24 23:00:00
##           loc              class
##       1: loc1 loc1 DSWRF_surface
##       2: loc2 loc2 DSWRF_surface
##       3: loc3 loc3 DSWRF_surface
##       4: loc4 loc4 DSWRF_surface
##       5: loc5 loc5 DSWRF_surface
##      ---                        
## 1129964: loc3      loc3 wdir_10m
## 1129965: loc4      loc4 wdir_10m
## 1129966: loc5      loc5 wdir_10m
## 1129967: loc6      loc6 wdir_10m
## 1129968: loc7      loc7 wdir_10m
w2 = w1[,list(datetime,class,value)]
w2[,class:=as.factor(class)]
w2
##                     datetime              class value
##       1: 2018-12-31 00:00:00 loc1 DSWRF_surface     0
##       2: 2018-12-31 00:00:00 loc2 DSWRF_surface     0
##       3: 2018-12-31 00:00:00 loc3 DSWRF_surface     0
##       4: 2018-12-31 00:00:00 loc4 DSWRF_surface     0
##       5: 2018-12-31 00:00:00 loc5 DSWRF_surface     0
##      ---                                             
## 1129964: 2022-01-24 23:00:00      loc3 wdir_10m    NA
## 1129965: 2022-01-24 23:00:00      loc4 wdir_10m    NA
## 1129966: 2022-01-24 23:00:00      loc5 wdir_10m    NA
## 1129967: 2022-01-24 23:00:00      loc6 wdir_10m    NA
## 1129968: 2022-01-24 23:00:00      loc7 wdir_10m    NA
trainw2=dcast(w2[!is.na(value)],datetime~class,value.var='value')
trainw2
##                   datetime loc1 DSWRF_surface loc1 RH_2.m.above.ground
##     1: 2018-12-31 00:00:00               0.00                 83.30000
##     2: 2018-12-31 01:00:00               0.00                 82.00000
##     3: 2018-12-31 02:00:00               0.00                 80.70000
##     4: 2018-12-31 03:00:00               0.00                 79.40000
##     5: 2018-12-31 04:00:00               0.00                 78.83333
##    ---                                                                
## 26898: 2022-01-24 17:00:00             348.20                 75.60000
## 26899: 2022-01-24 18:00:00             255.46                 79.00000
## 26900: 2022-01-24 19:00:00             191.60                 78.00000
## 26901: 2022-01-24 20:00:00             153.28                 80.00000
## 26902: 2022-01-24 21:00:00             127.74                 79.90000
##        loc1 TCDC_low.cloud.layer loc1 TMP_2.m.above.ground loc1 wdir_10m
##     1:                       0.0                 -8.284000      219.0959
##     2:                       1.0                 -8.104667      217.8237
##     3:                       2.0                 -7.925333      216.5515
##     4:                       3.0                 -7.746000      215.2793
##     5:                       5.0                 -7.995333      216.4846
##    ---                                                                  
## 26898:                       3.4                 -8.619000      232.9766
## 26899:                       2.4                -13.750000      258.6988
## 26900:                       1.8                -15.312000      261.6345
## 26901:                       1.5                -15.771000      262.4866
## 26902:                       1.2                -15.815000      264.9919
##        loc1 ws_10m loc2 DSWRF_surface loc2 RH_2.m.above.ground
##     1:    2.790017               0.00                     76.1
##     2:    2.725563               0.00                     74.6
##     3:    2.661109               0.00                     73.1
##     4:    2.596655               0.00                     71.6
##     5:    2.551815               0.00                     71.3
##    ---                                                        
## 26898:    1.541491             278.72                     69.0
## 26899:    2.451014             197.54                     65.9
## 26900:    2.975670             148.14                     67.6
## 26901:    3.469357             118.52                     73.1
## 26902:    3.329440              98.78                     76.8
##        loc2 TCDC_low.cloud.layer loc2 TMP_2.m.above.ground loc2 wdir_10m
##     1:                  66.00000                  4.396000      179.5000
##     2:                  59.66667                  3.972000      182.0823
##     3:                  53.33333                  3.548000      184.6647
##     4:                  47.00000                  3.124000      187.2471
##     5:                  31.66667                  3.111333      186.7205
##    ---                                                                  
## 26898:                   0.00000                  3.611000      163.0320
## 26899:                   0.00000                  1.030000      168.9420
## 26900:                   0.00000                  0.198000      169.6947
## 26901:                   0.00000                 -0.651000      168.4363
## 26902:                   0.00000                 -1.405000      181.3601
##        loc2 ws_10m loc3 DSWRF_surface loc3 RH_2.m.above.ground
##     1:   2.3553997               0.00                 93.20000
##     2:   2.3094280               0.00                 92.93333
##     3:   2.2634562               0.00                 92.66667
##     4:   2.2174845               0.00                 92.40000
##     5:   2.2073096               0.00                 92.06667
##    ---                                                        
## 26898:   0.3744988             308.36                 81.10000
## 26899:   0.3467570             222.70                 85.70000
## 26900:   1.1514952             167.02                 86.10000
## 26901:   2.1472339             133.62                 85.80000
## 26902:   2.8114421             111.34                 84.90000
##        loc3 TCDC_low.cloud.layer loc3 TMP_2.m.above.ground loc3 wdir_10m
##     1:                      95.0                 -4.684000      231.8005
##     2:                      89.0                 -4.884667      245.0924
##     3:                      83.0                 -5.085333      258.3843
##     4:                      77.0                 -5.286000      271.6762
##     5:                      54.0                 -5.995333      217.6031
##    ---                                                                  
## 26898:                      79.6                 -8.829000      188.7512
## 26899:                      77.2                -12.350000      148.3612
## 26900:                      74.2                -13.982000      127.2406
## 26901:                      72.7                -14.521000      120.5353
## 26902:                      73.8                -14.465000      116.5458
##        loc3 ws_10m loc4 DSWRF_surface loc4 RH_2.m.above.ground
##     1:   0.8009606               0.00                 75.20000
##     2:   0.6506007               0.00                 76.03333
##     3:   0.5002407               0.00                 76.86667
##     4:   0.3498807               0.00                 77.70000
##     5:   0.3796237               0.00                 77.60000
##    ---                                                        
## 26898:   2.4365654             190.82                 63.50000
## 26899:   2.6431705             144.14                 72.80000
## 26900:   2.8966046             108.96                 77.00000
## 26901:   2.8217623              87.18                 76.60000
## 26902:   2.6194151              72.64                 74.70000
##        loc4 TCDC_low.cloud.layer loc4 TMP_2.m.above.ground loc4 wdir_10m
##     1:                         0                  5.026000      253.8836
##     2:                         0                  4.732000      257.6694
##     3:                         0                  4.438000      261.4552
##     4:                         0                  4.144000      265.2409
##     5:                         0                  4.061333      265.6834
##    ---                                                                  
## 26898:                       100                  0.231000      238.7585
## 26899:                       100                 -0.710000      240.8043
## 26900:                       100                 -1.482000      238.6160
## 26901:                       100                 -1.801000      226.3945
## 26902:                       100                 -1.765000      233.5635
##        loc4 ws_10m loc5 DSWRF_surface loc5 RH_2.m.above.ground
##     1:    2.684959               0.00                 88.70000
##     2:    2.672927               0.00                 88.53333
##     3:    2.660895               0.00                 88.36667
##     4:    2.648862               0.00                 88.20000
##     5:    2.636125               0.00                 87.46667
##    ---                                                        
## 26898:    4.913175             302.88                 80.20000
## 26899:    4.551752             223.22                 87.40000
## 26900:    4.268511             167.44                 91.10000
## 26901:    3.659096             133.94                 93.70000
## 26902:    3.637878             111.62                 94.90000
##        loc5 TCDC_low.cloud.layer loc5 TMP_2.m.above.ground loc5 wdir_10m
##     1:                 2.0000000                 -3.174000     336.58831
##     2:                 1.3333333                 -3.294667     341.74223
##     3:                 0.6666667                 -3.415333     346.89616
##     4:                 0.0000000                 -3.536000     352.05008
##     5:                 0.0000000                 -3.878667     349.81053
##    ---                                                                  
## 26898:                55.4000000                 -6.329000     165.58548
## 26899:                38.6000000                -10.300000     172.49167
## 26900:                31.0000000                -12.952000     148.07612
## 26901:                26.0000000                -14.521000     100.22730
## 26902:                22.5000000                -15.455000      82.61275
##        loc5 ws_10m loc6 DSWRF_surface loc6 RH_2.m.above.ground
##     1:   0.7788009               0.00                 94.20000
##     2:   1.0005702               0.00                 94.10000
##     3:   1.2223395               0.00                 94.00000
##     4:   1.4441088               0.00                 93.90000
##     5:   1.5497829               0.00                 93.63333
##    ---                                                        
## 26898:   2.1663980             196.60                 89.50000
## 26899:   1.8864951             141.86                 94.90000
## 26900:   1.2405518             106.40                 96.40000
## 26901:   1.3722332              85.12                 96.50000
## 26902:   1.9393671              70.94                 96.70000
##        loc6 TCDC_low.cloud.layer loc6 TMP_2.m.above.ground loc6 wdir_10m
##     1:                  11.00000                 -6.874000      93.47636
##     2:                  12.00000                 -7.021333      90.29830
##     3:                  13.00000                 -7.168667      87.12025
##     4:                  14.00000                 -7.316000      83.94219
##     5:                  10.66667                 -7.545333      93.91665
##    ---                                                                  
## 26898:                  97.50000                 -7.649000      62.27623
## 26899:                  93.10000                -10.020000      45.36578
## 26900:                  93.10000                -10.882000      36.68694
## 26901:                  92.70000                -11.191000      19.84014
## 26902:                  93.30000                -11.685000     357.44609
##        loc6 ws_10m loc7 DSWRF_surface loc7 RH_2.m.above.ground
##     1:   0.9122336               0.00                 76.20000
##     2:   0.8931703               0.00                 75.53333
##     3:   0.8741070               0.00                 74.86667
##     4:   0.8550436               0.00                 74.20000
##     5:   0.7646836               0.00                 73.43333
##    ---                                                        
## 26898:   2.9702756             235.86                 59.30000
## 26899:   1.8641058             174.56                 61.00000
## 26900:   1.0812659             131.02                 61.10000
## 26901:   0.7084035             104.80                 59.00000
## 26902:   0.5999499              87.34                 58.30000
##        loc7 TCDC_low.cloud.layer loc7 TMP_2.m.above.ground loc7 wdir_10m
##     1:                  83.00000                  6.886000      288.4040
##     2:                  76.33333                  6.808667      296.8295
##     3:                  69.66667                  6.731333      305.2550
##     4:                  63.00000                  6.654000      313.6806
##     5:                  57.66667                  6.551333      308.5280
##    ---                                                                  
## 26898:                  43.80000                  1.081000      180.9141
## 26899:                  32.70000                  0.560000      181.5385
## 26900:                  37.30000                  0.198000      182.9638
## 26901:                  49.10000                  0.289000      187.1146
## 26902:                  57.40000                  0.275000      190.2860
##        loc7 ws_10m
##     1:    3.150579
##     2:    3.118881
##     3:    3.087182
##     4:    3.055483
##     5:    2.920705
##    ---            
## 26898:    5.058844
## 26899:    4.972112
## 26900:    4.719232
## 26901:    4.840923
## 26902:    4.685959
names(trainw2) <- gsub(" ", "_", names(trainw2))
trainw2
##                   datetime loc1_DSWRF_surface loc1_RH_2.m.above.ground
##     1: 2018-12-31 00:00:00               0.00                 83.30000
##     2: 2018-12-31 01:00:00               0.00                 82.00000
##     3: 2018-12-31 02:00:00               0.00                 80.70000
##     4: 2018-12-31 03:00:00               0.00                 79.40000
##     5: 2018-12-31 04:00:00               0.00                 78.83333
##    ---                                                                
## 26898: 2022-01-24 17:00:00             348.20                 75.60000
## 26899: 2022-01-24 18:00:00             255.46                 79.00000
## 26900: 2022-01-24 19:00:00             191.60                 78.00000
## 26901: 2022-01-24 20:00:00             153.28                 80.00000
## 26902: 2022-01-24 21:00:00             127.74                 79.90000
##        loc1_TCDC_low.cloud.layer loc1_TMP_2.m.above.ground loc1_wdir_10m
##     1:                       0.0                 -8.284000      219.0959
##     2:                       1.0                 -8.104667      217.8237
##     3:                       2.0                 -7.925333      216.5515
##     4:                       3.0                 -7.746000      215.2793
##     5:                       5.0                 -7.995333      216.4846
##    ---                                                                  
## 26898:                       3.4                 -8.619000      232.9766
## 26899:                       2.4                -13.750000      258.6988
## 26900:                       1.8                -15.312000      261.6345
## 26901:                       1.5                -15.771000      262.4866
## 26902:                       1.2                -15.815000      264.9919
##        loc1_ws_10m loc2_DSWRF_surface loc2_RH_2.m.above.ground
##     1:    2.790017               0.00                     76.1
##     2:    2.725563               0.00                     74.6
##     3:    2.661109               0.00                     73.1
##     4:    2.596655               0.00                     71.6
##     5:    2.551815               0.00                     71.3
##    ---                                                        
## 26898:    1.541491             278.72                     69.0
## 26899:    2.451014             197.54                     65.9
## 26900:    2.975670             148.14                     67.6
## 26901:    3.469357             118.52                     73.1
## 26902:    3.329440              98.78                     76.8
##        loc2_TCDC_low.cloud.layer loc2_TMP_2.m.above.ground loc2_wdir_10m
##     1:                  66.00000                  4.396000      179.5000
##     2:                  59.66667                  3.972000      182.0823
##     3:                  53.33333                  3.548000      184.6647
##     4:                  47.00000                  3.124000      187.2471
##     5:                  31.66667                  3.111333      186.7205
##    ---                                                                  
## 26898:                   0.00000                  3.611000      163.0320
## 26899:                   0.00000                  1.030000      168.9420
## 26900:                   0.00000                  0.198000      169.6947
## 26901:                   0.00000                 -0.651000      168.4363
## 26902:                   0.00000                 -1.405000      181.3601
##        loc2_ws_10m loc3_DSWRF_surface loc3_RH_2.m.above.ground
##     1:   2.3553997               0.00                 93.20000
##     2:   2.3094280               0.00                 92.93333
##     3:   2.2634562               0.00                 92.66667
##     4:   2.2174845               0.00                 92.40000
##     5:   2.2073096               0.00                 92.06667
##    ---                                                        
## 26898:   0.3744988             308.36                 81.10000
## 26899:   0.3467570             222.70                 85.70000
## 26900:   1.1514952             167.02                 86.10000
## 26901:   2.1472339             133.62                 85.80000
## 26902:   2.8114421             111.34                 84.90000
##        loc3_TCDC_low.cloud.layer loc3_TMP_2.m.above.ground loc3_wdir_10m
##     1:                      95.0                 -4.684000      231.8005
##     2:                      89.0                 -4.884667      245.0924
##     3:                      83.0                 -5.085333      258.3843
##     4:                      77.0                 -5.286000      271.6762
##     5:                      54.0                 -5.995333      217.6031
##    ---                                                                  
## 26898:                      79.6                 -8.829000      188.7512
## 26899:                      77.2                -12.350000      148.3612
## 26900:                      74.2                -13.982000      127.2406
## 26901:                      72.7                -14.521000      120.5353
## 26902:                      73.8                -14.465000      116.5458
##        loc3_ws_10m loc4_DSWRF_surface loc4_RH_2.m.above.ground
##     1:   0.8009606               0.00                 75.20000
##     2:   0.6506007               0.00                 76.03333
##     3:   0.5002407               0.00                 76.86667
##     4:   0.3498807               0.00                 77.70000
##     5:   0.3796237               0.00                 77.60000
##    ---                                                        
## 26898:   2.4365654             190.82                 63.50000
## 26899:   2.6431705             144.14                 72.80000
## 26900:   2.8966046             108.96                 77.00000
## 26901:   2.8217623              87.18                 76.60000
## 26902:   2.6194151              72.64                 74.70000
##        loc4_TCDC_low.cloud.layer loc4_TMP_2.m.above.ground loc4_wdir_10m
##     1:                         0                  5.026000      253.8836
##     2:                         0                  4.732000      257.6694
##     3:                         0                  4.438000      261.4552
##     4:                         0                  4.144000      265.2409
##     5:                         0                  4.061333      265.6834
##    ---                                                                  
## 26898:                       100                  0.231000      238.7585
## 26899:                       100                 -0.710000      240.8043
## 26900:                       100                 -1.482000      238.6160
## 26901:                       100                 -1.801000      226.3945
## 26902:                       100                 -1.765000      233.5635
##        loc4_ws_10m loc5_DSWRF_surface loc5_RH_2.m.above.ground
##     1:    2.684959               0.00                 88.70000
##     2:    2.672927               0.00                 88.53333
##     3:    2.660895               0.00                 88.36667
##     4:    2.648862               0.00                 88.20000
##     5:    2.636125               0.00                 87.46667
##    ---                                                        
## 26898:    4.913175             302.88                 80.20000
## 26899:    4.551752             223.22                 87.40000
## 26900:    4.268511             167.44                 91.10000
## 26901:    3.659096             133.94                 93.70000
## 26902:    3.637878             111.62                 94.90000
##        loc5_TCDC_low.cloud.layer loc5_TMP_2.m.above.ground loc5_wdir_10m
##     1:                 2.0000000                 -3.174000     336.58831
##     2:                 1.3333333                 -3.294667     341.74223
##     3:                 0.6666667                 -3.415333     346.89616
##     4:                 0.0000000                 -3.536000     352.05008
##     5:                 0.0000000                 -3.878667     349.81053
##    ---                                                                  
## 26898:                55.4000000                 -6.329000     165.58548
## 26899:                38.6000000                -10.300000     172.49167
## 26900:                31.0000000                -12.952000     148.07612
## 26901:                26.0000000                -14.521000     100.22730
## 26902:                22.5000000                -15.455000      82.61275
##        loc5_ws_10m loc6_DSWRF_surface loc6_RH_2.m.above.ground
##     1:   0.7788009               0.00                 94.20000
##     2:   1.0005702               0.00                 94.10000
##     3:   1.2223395               0.00                 94.00000
##     4:   1.4441088               0.00                 93.90000
##     5:   1.5497829               0.00                 93.63333
##    ---                                                        
## 26898:   2.1663980             196.60                 89.50000
## 26899:   1.8864951             141.86                 94.90000
## 26900:   1.2405518             106.40                 96.40000
## 26901:   1.3722332              85.12                 96.50000
## 26902:   1.9393671              70.94                 96.70000
##        loc6_TCDC_low.cloud.layer loc6_TMP_2.m.above.ground loc6_wdir_10m
##     1:                  11.00000                 -6.874000      93.47636
##     2:                  12.00000                 -7.021333      90.29830
##     3:                  13.00000                 -7.168667      87.12025
##     4:                  14.00000                 -7.316000      83.94219
##     5:                  10.66667                 -7.545333      93.91665
##    ---                                                                  
## 26898:                  97.50000                 -7.649000      62.27623
## 26899:                  93.10000                -10.020000      45.36578
## 26900:                  93.10000                -10.882000      36.68694
## 26901:                  92.70000                -11.191000      19.84014
## 26902:                  93.30000                -11.685000     357.44609
##        loc6_ws_10m loc7_DSWRF_surface loc7_RH_2.m.above.ground
##     1:   0.9122336               0.00                 76.20000
##     2:   0.8931703               0.00                 75.53333
##     3:   0.8741070               0.00                 74.86667
##     4:   0.8550436               0.00                 74.20000
##     5:   0.7646836               0.00                 73.43333
##    ---                                                        
## 26898:   2.9702756             235.86                 59.30000
## 26899:   1.8641058             174.56                 61.00000
## 26900:   1.0812659             131.02                 61.10000
## 26901:   0.7084035             104.80                 59.00000
## 26902:   0.5999499              87.34                 58.30000
##        loc7_TCDC_low.cloud.layer loc7_TMP_2.m.above.ground loc7_wdir_10m
##     1:                  83.00000                  6.886000      288.4040
##     2:                  76.33333                  6.808667      296.8295
##     3:                  69.66667                  6.731333      305.2550
##     4:                  63.00000                  6.654000      313.6806
##     5:                  57.66667                  6.551333      308.5280
##    ---                                                                  
## 26898:                  43.80000                  1.081000      180.9141
## 26899:                  32.70000                  0.560000      181.5385
## 26900:                  37.30000                  0.198000      182.9638
## 26901:                  49.10000                  0.289000      187.1146
## 26902:                  57.40000                  0.275000      190.2860
##        loc7_ws_10m
##     1:    3.150579
##     2:    3.118881
##     3:    3.087182
##     4:    3.055483
##     5:    2.920705
##    ---            
## 26898:    5.058844
## 26899:    4.972112
## 26900:    4.719232
## 26901:    4.840923
## 26902:    4.685959
all = merge(c2,trainw2, by="datetime")
all
##                   datetime       net system_direction loc1_DSWRF_surface
##     1: 2019-01-01 00:00:00 -1934.795         Negative               0.00
##     2: 2019-01-01 01:00:00  -750.866         Negative               0.00
##     3: 2019-01-01 02:00:00  -982.767         Negative               0.00
##     4: 2019-01-01 03:00:00  -764.620         Negative               0.00
##     5: 2019-01-01 04:00:00 -1233.289         Negative               0.00
##    ---                                                                  
## 26804: 2022-01-21 19:00:00     0.000                               55.38
## 26805: 2022-01-21 20:00:00     0.000                               44.30
## 26806: 2022-01-21 21:00:00     0.000                               36.92
## 26807: 2022-01-21 22:00:00     0.000                                0.00
## 26808: 2022-01-21 23:00:00     0.000                                0.00
##        loc1_RH_2.m.above.ground loc1_TCDC_low.cloud.layer
##     1:                 84.20000                   0.00000
##     2:                 84.43333                   3.00000
##     3:                 84.66667                   6.00000
##     4:                 84.90000                   9.00000
##     5:                 87.26667                  29.66667
##    ---                                                   
## 26804:                 97.90000                  99.90000
## 26805:                 98.00000                  99.90000
## 26806:                 98.70000                  99.90000
## 26807:                 98.80000                 100.00000
## 26808:                 98.90000                 100.00000
##        loc1_TMP_2.m.above.ground loc1_wdir_10m loc1_ws_10m loc2_DSWRF_surface
##     1:                    -4.081    244.986564    2.302671               0.00
##     2:                    -3.707    245.382871    2.443606               0.00
##     3:                    -3.333    245.779178    2.584541               0.00
##     4:                    -2.959    246.175485    2.725476               0.00
##     5:                    -2.591    240.651199    2.755832               0.00
##    ---                                                                       
## 26804:                    -1.623      2.212407    1.948613              40.46
## 26805:                    -1.285    357.806092    1.767906              32.36
## 26806:                    -0.937    354.741987    2.657864              26.98
## 26807:                    -0.641      7.030764    3.098056               0.00
## 26808:                    -0.540     27.723707    3.520917               0.00
##        loc2_RH_2.m.above.ground loc2_TCDC_low.cloud.layer
##     1:                 59.80000                       0.0
##     2:                 56.93333                       0.0
##     3:                 54.06667                       0.0
##     4:                 51.20000                       0.0
##     5:                 51.56667                       0.0
##    ---                                                   
## 26804:                 66.80000                      99.9
## 26805:                 68.60000                      99.9
## 26806:                 68.50000                      99.9
## 26807:                 73.90000                     100.0
## 26808:                 79.80000                     100.0
##        loc2_TMP_2.m.above.ground loc2_wdir_10m loc2_ws_10m loc3_DSWRF_surface
##     1:                  3.369000      192.5959    3.057220               0.00
##     2:                  3.333000      192.2694    3.202298               0.00
##     3:                  3.297000      191.9429    3.347376               0.00
##     4:                  3.261000      191.6164    3.492454               0.00
##     5:                  3.365667      191.7771    3.608525               0.00
##    ---                                                                       
## 26804:                  4.347000      208.0302    3.265933             163.96
## 26805:                  4.295000      203.6774    3.355885             131.16
## 26806:                  4.533000      206.7301    3.698566             109.30
## 26807:                  4.649000      211.2813    4.276995               0.00
## 26808:                  4.530000      212.4927    3.987479               0.00
##        loc3_RH_2.m.above.ground loc3_TCDC_low.cloud.layer
##     1:                 84.40000                 1.0000000
##     2:                 85.06667                 0.6666667
##     3:                 85.73333                 0.3333333
##     4:                 86.40000                 0.0000000
##     5:                 87.83333                 1.3333333
##    ---                                                   
## 26804:                 72.40000                 0.0000000
## 26805:                 71.90000                 0.0000000
## 26806:                 72.70000                 0.0000000
## 26807:                 74.60000                 5.0000000
## 26808:                 77.70000                 4.8000000
##        loc3_TMP_2.m.above.ground loc3_wdir_10m loc3_ws_10m loc4_DSWRF_surface
##     1:                 -6.211000     351.73126    1.784915               0.00
##     2:                 -5.950333     350.11411    1.928927               0.00
##     3:                 -5.689667     348.49697    2.072939               0.00
##     4:                 -5.429000     346.87983    2.216951               0.00
##     5:                 -5.094333     337.72332    1.997965               0.00
##    ---                                                                       
## 26804:                 -2.723000      43.76432    3.014659              30.56
## 26805:                 -3.055000      45.90085    2.955315              24.46
## 26806:                 -3.107000      42.20943    2.763192              20.36
## 26807:                 -3.091000      39.53944    2.716282               0.00
## 26808:                 -2.910000      51.42887    2.849656               0.00
##        loc4_RH_2.m.above.ground loc4_TCDC_low.cloud.layer
##     1:                 52.20000                   0.00000
##     2:                 53.90000                   9.00000
##     3:                 55.60000                  18.00000
##     4:                 57.30000                  27.00000
##     5:                 59.76667                  51.33333
##    ---                                                   
## 26804:                 88.20000                  98.10000
## 26805:                 89.20000                  98.50000
## 26806:                 89.20000                  98.70000
## 26807:                 89.30000                 100.00000
## 26808:                 90.10000                 100.00000
##        loc4_TMP_2.m.above.ground loc4_wdir_10m loc4_ws_10m loc5_DSWRF_surface
##     1:                  7.309000      248.9115    5.151732               0.00
##     2:                  7.203000      248.8854    5.512422               0.00
##     3:                  7.097000      248.8594    5.873112               0.00
##     4:                  6.991000      248.8333    6.233802               0.00
##     5:                  6.902333      248.9605    6.574972               0.00
##    ---                                                                       
## 26804:                  9.287000      352.6565    4.262119             116.34
## 26805:                  9.265000      352.4482    4.243415              93.08
## 26806:                  9.233000      350.1066    4.036709              77.56
## 26807:                  9.189000      346.2419    3.577402               0.00
## 26808:                  9.140000      339.1743    3.634154               0.00
##        loc5_RH_2.m.above.ground loc5_TCDC_low.cloud.layer
##     1:                 88.30000                 0.0000000
##     2:                 87.66667                 0.0000000
##     3:                 87.03333                 0.0000000
##     4:                 86.40000                 0.0000000
##     5:                 86.66667                 0.3333333
##    ---                                                   
## 26804:                 84.30000                69.8000000
## 26805:                 81.30000                69.5000000
## 26806:                 86.40000                70.2000000
## 26807:                 89.50000               100.0000000
## 26808:                 94.00000                99.1000000
##        loc5_TMP_2.m.above.ground loc5_wdir_10m loc5_ws_10m loc6_DSWRF_surface
##     1:                 -4.191000     297.65206    3.913716               0.00
##     2:                 -4.070333     295.48408    4.243639               0.00
##     3:                 -3.949667     293.31610    4.573562               0.00
##     4:                 -3.829000     291.14811    4.903485               0.00
##     5:                 -3.864333     290.61532    4.996934               0.00
##    ---                                                                       
## 26804:                  1.997000      19.05118    4.366314             114.36
## 26805:                  2.185000      29.69346    5.556206              91.48
## 26806:                  1.463000      47.68443    4.525578              76.24
## 26807:                  1.679000      63.83293    3.843080               0.00
## 26808:                  1.490000      56.89650    3.948929               0.00
##        loc6_RH_2.m.above.ground loc6_TCDC_low.cloud.layer
##     1:                 92.10000                  28.00000
##     2:                 92.50000                  25.66667
##     3:                 92.90000                  23.33333
##     4:                 93.30000                  21.00000
##     5:                 94.03333                  21.00000
##    ---                                                   
## 26804:                 73.20000                  74.30000
## 26805:                 72.20000                  79.40000
## 26806:                 74.20000                  82.60000
## 26807:                 89.20000                 100.00000
## 26808:                 82.50000                 100.00000
##        loc6_TMP_2.m.above.ground loc6_wdir_10m loc6_ws_10m loc7_DSWRF_surface
##     1:                 -7.041000      280.4415    1.745607               0.00
##     2:                 -7.200333      274.3150    1.962468               0.00
##     3:                 -7.359667      268.1885    2.179329               0.00
##     4:                 -7.519000      262.0619    2.396190               0.00
##     5:                 -7.527667      260.3688    2.522906               0.00
##    ---                                                                       
## 26804:                 -1.953000      324.6584    2.117349               7.32
## 26805:                 -1.895000      312.2008    2.332197               5.86
## 26806:                 -1.907000      331.4124    2.160014               4.88
## 26807:                 -2.371000      300.8462    2.388677               0.00
## 26808:                 -1.210000      345.0344    2.563677               0.00
##        loc7_RH_2.m.above.ground loc7_TCDC_low.cloud.layer
##     1:                 65.80000                         0
##     2:                 66.40000                         0
##     3:                 67.00000                         0
##     4:                 67.60000                         0
##     5:                 67.83333                         0
##    ---                                                   
## 26804:                 94.90000                       100
## 26805:                 94.80000                       100
## 26806:                 95.00000                       100
## 26807:                 95.20000                       100
## 26808:                 95.20000                       100
##        loc7_TMP_2.m.above.ground loc7_wdir_10m loc7_ws_10m
##     1:                  7.239000      250.5319    5.861836
##     2:                  7.153000      250.9196    6.254371
##     3:                  7.067000      251.3073    6.646906
##     4:                  6.981000      251.6950    7.039441
##     5:                  6.962333      252.8526    7.291198
##    ---                                                    
## 26804:                  3.577000      192.1873    6.417472
## 26805:                  2.815000      181.7018    6.656326
## 26806:                  2.743000      185.1566    6.047797
## 26807:                  2.399000      184.1567    6.081236
## 26808:                  2.180000      182.7795    6.641093
ggplot(all ,aes(x=datetime,y=net)) + geom_line()

all[,trnd:=1:.N]

all[,w_day:=as.character(wday(datetime,label=T))]
all[,mon:=as.character(month(datetime,label=T))]
all[,hour:=as.factor(hour(datetime))]
all[,Date:=as.Date(datetime)]
all
##                   datetime       net system_direction loc1_DSWRF_surface
##     1: 2019-01-01 00:00:00 -1934.795         Negative               0.00
##     2: 2019-01-01 01:00:00  -750.866         Negative               0.00
##     3: 2019-01-01 02:00:00  -982.767         Negative               0.00
##     4: 2019-01-01 03:00:00  -764.620         Negative               0.00
##     5: 2019-01-01 04:00:00 -1233.289         Negative               0.00
##    ---                                                                  
## 26804: 2022-01-21 19:00:00     0.000                               55.38
## 26805: 2022-01-21 20:00:00     0.000                               44.30
## 26806: 2022-01-21 21:00:00     0.000                               36.92
## 26807: 2022-01-21 22:00:00     0.000                                0.00
## 26808: 2022-01-21 23:00:00     0.000                                0.00
##        loc1_RH_2.m.above.ground loc1_TCDC_low.cloud.layer
##     1:                 84.20000                   0.00000
##     2:                 84.43333                   3.00000
##     3:                 84.66667                   6.00000
##     4:                 84.90000                   9.00000
##     5:                 87.26667                  29.66667
##    ---                                                   
## 26804:                 97.90000                  99.90000
## 26805:                 98.00000                  99.90000
## 26806:                 98.70000                  99.90000
## 26807:                 98.80000                 100.00000
## 26808:                 98.90000                 100.00000
##        loc1_TMP_2.m.above.ground loc1_wdir_10m loc1_ws_10m loc2_DSWRF_surface
##     1:                    -4.081    244.986564    2.302671               0.00
##     2:                    -3.707    245.382871    2.443606               0.00
##     3:                    -3.333    245.779178    2.584541               0.00
##     4:                    -2.959    246.175485    2.725476               0.00
##     5:                    -2.591    240.651199    2.755832               0.00
##    ---                                                                       
## 26804:                    -1.623      2.212407    1.948613              40.46
## 26805:                    -1.285    357.806092    1.767906              32.36
## 26806:                    -0.937    354.741987    2.657864              26.98
## 26807:                    -0.641      7.030764    3.098056               0.00
## 26808:                    -0.540     27.723707    3.520917               0.00
##        loc2_RH_2.m.above.ground loc2_TCDC_low.cloud.layer
##     1:                 59.80000                       0.0
##     2:                 56.93333                       0.0
##     3:                 54.06667                       0.0
##     4:                 51.20000                       0.0
##     5:                 51.56667                       0.0
##    ---                                                   
## 26804:                 66.80000                      99.9
## 26805:                 68.60000                      99.9
## 26806:                 68.50000                      99.9
## 26807:                 73.90000                     100.0
## 26808:                 79.80000                     100.0
##        loc2_TMP_2.m.above.ground loc2_wdir_10m loc2_ws_10m loc3_DSWRF_surface
##     1:                  3.369000      192.5959    3.057220               0.00
##     2:                  3.333000      192.2694    3.202298               0.00
##     3:                  3.297000      191.9429    3.347376               0.00
##     4:                  3.261000      191.6164    3.492454               0.00
##     5:                  3.365667      191.7771    3.608525               0.00
##    ---                                                                       
## 26804:                  4.347000      208.0302    3.265933             163.96
## 26805:                  4.295000      203.6774    3.355885             131.16
## 26806:                  4.533000      206.7301    3.698566             109.30
## 26807:                  4.649000      211.2813    4.276995               0.00
## 26808:                  4.530000      212.4927    3.987479               0.00
##        loc3_RH_2.m.above.ground loc3_TCDC_low.cloud.layer
##     1:                 84.40000                 1.0000000
##     2:                 85.06667                 0.6666667
##     3:                 85.73333                 0.3333333
##     4:                 86.40000                 0.0000000
##     5:                 87.83333                 1.3333333
##    ---                                                   
## 26804:                 72.40000                 0.0000000
## 26805:                 71.90000                 0.0000000
## 26806:                 72.70000                 0.0000000
## 26807:                 74.60000                 5.0000000
## 26808:                 77.70000                 4.8000000
##        loc3_TMP_2.m.above.ground loc3_wdir_10m loc3_ws_10m loc4_DSWRF_surface
##     1:                 -6.211000     351.73126    1.784915               0.00
##     2:                 -5.950333     350.11411    1.928927               0.00
##     3:                 -5.689667     348.49697    2.072939               0.00
##     4:                 -5.429000     346.87983    2.216951               0.00
##     5:                 -5.094333     337.72332    1.997965               0.00
##    ---                                                                       
## 26804:                 -2.723000      43.76432    3.014659              30.56
## 26805:                 -3.055000      45.90085    2.955315              24.46
## 26806:                 -3.107000      42.20943    2.763192              20.36
## 26807:                 -3.091000      39.53944    2.716282               0.00
## 26808:                 -2.910000      51.42887    2.849656               0.00
##        loc4_RH_2.m.above.ground loc4_TCDC_low.cloud.layer
##     1:                 52.20000                   0.00000
##     2:                 53.90000                   9.00000
##     3:                 55.60000                  18.00000
##     4:                 57.30000                  27.00000
##     5:                 59.76667                  51.33333
##    ---                                                   
## 26804:                 88.20000                  98.10000
## 26805:                 89.20000                  98.50000
## 26806:                 89.20000                  98.70000
## 26807:                 89.30000                 100.00000
## 26808:                 90.10000                 100.00000
##        loc4_TMP_2.m.above.ground loc4_wdir_10m loc4_ws_10m loc5_DSWRF_surface
##     1:                  7.309000      248.9115    5.151732               0.00
##     2:                  7.203000      248.8854    5.512422               0.00
##     3:                  7.097000      248.8594    5.873112               0.00
##     4:                  6.991000      248.8333    6.233802               0.00
##     5:                  6.902333      248.9605    6.574972               0.00
##    ---                                                                       
## 26804:                  9.287000      352.6565    4.262119             116.34
## 26805:                  9.265000      352.4482    4.243415              93.08
## 26806:                  9.233000      350.1066    4.036709              77.56
## 26807:                  9.189000      346.2419    3.577402               0.00
## 26808:                  9.140000      339.1743    3.634154               0.00
##        loc5_RH_2.m.above.ground loc5_TCDC_low.cloud.layer
##     1:                 88.30000                 0.0000000
##     2:                 87.66667                 0.0000000
##     3:                 87.03333                 0.0000000
##     4:                 86.40000                 0.0000000
##     5:                 86.66667                 0.3333333
##    ---                                                   
## 26804:                 84.30000                69.8000000
## 26805:                 81.30000                69.5000000
## 26806:                 86.40000                70.2000000
## 26807:                 89.50000               100.0000000
## 26808:                 94.00000                99.1000000
##        loc5_TMP_2.m.above.ground loc5_wdir_10m loc5_ws_10m loc6_DSWRF_surface
##     1:                 -4.191000     297.65206    3.913716               0.00
##     2:                 -4.070333     295.48408    4.243639               0.00
##     3:                 -3.949667     293.31610    4.573562               0.00
##     4:                 -3.829000     291.14811    4.903485               0.00
##     5:                 -3.864333     290.61532    4.996934               0.00
##    ---                                                                       
## 26804:                  1.997000      19.05118    4.366314             114.36
## 26805:                  2.185000      29.69346    5.556206              91.48
## 26806:                  1.463000      47.68443    4.525578              76.24
## 26807:                  1.679000      63.83293    3.843080               0.00
## 26808:                  1.490000      56.89650    3.948929               0.00
##        loc6_RH_2.m.above.ground loc6_TCDC_low.cloud.layer
##     1:                 92.10000                  28.00000
##     2:                 92.50000                  25.66667
##     3:                 92.90000                  23.33333
##     4:                 93.30000                  21.00000
##     5:                 94.03333                  21.00000
##    ---                                                   
## 26804:                 73.20000                  74.30000
## 26805:                 72.20000                  79.40000
## 26806:                 74.20000                  82.60000
## 26807:                 89.20000                 100.00000
## 26808:                 82.50000                 100.00000
##        loc6_TMP_2.m.above.ground loc6_wdir_10m loc6_ws_10m loc7_DSWRF_surface
##     1:                 -7.041000      280.4415    1.745607               0.00
##     2:                 -7.200333      274.3150    1.962468               0.00
##     3:                 -7.359667      268.1885    2.179329               0.00
##     4:                 -7.519000      262.0619    2.396190               0.00
##     5:                 -7.527667      260.3688    2.522906               0.00
##    ---                                                                       
## 26804:                 -1.953000      324.6584    2.117349               7.32
## 26805:                 -1.895000      312.2008    2.332197               5.86
## 26806:                 -1.907000      331.4124    2.160014               4.88
## 26807:                 -2.371000      300.8462    2.388677               0.00
## 26808:                 -1.210000      345.0344    2.563677               0.00
##        loc7_RH_2.m.above.ground loc7_TCDC_low.cloud.layer
##     1:                 65.80000                         0
##     2:                 66.40000                         0
##     3:                 67.00000                         0
##     4:                 67.60000                         0
##     5:                 67.83333                         0
##    ---                                                   
## 26804:                 94.90000                       100
## 26805:                 94.80000                       100
## 26806:                 95.00000                       100
## 26807:                 95.20000                       100
## 26808:                 95.20000                       100
##        loc7_TMP_2.m.above.ground loc7_wdir_10m loc7_ws_10m  trnd w_day mon hour
##     1:                  7.239000      250.5319    5.861836     1   Sal Oca    0
##     2:                  7.153000      250.9196    6.254371     2   Sal Oca    1
##     3:                  7.067000      251.3073    6.646906     3   Sal Oca    2
##     4:                  6.981000      251.6950    7.039441     4   Sal Oca    3
##     5:                  6.962333      252.8526    7.291198     5   Sal Oca    4
##    ---                                                                         
## 26804:                  3.577000      192.1873    6.417472 26804   Cum Oca   19
## 26805:                  2.815000      181.7018    6.656326 26805   Cum Oca   20
## 26806:                  2.743000      185.1566    6.047797 26806   Cum Oca   21
## 26807:                  2.399000      184.1567    6.081236 26807   Cum Oca   22
## 26808:                  2.180000      182.7795    6.641093 26808   Cum Oca   23
##              Date
##     1: 2019-01-01
##     2: 2019-01-01
##     3: 2019-01-01
##     4: 2019-01-01
##     5: 2019-01-01
##    ---           
## 26804: 2022-01-21
## 26805: 2022-01-21
## 26806: 2022-01-21
## 26807: 2022-01-21
## 26808: 2022-01-21

First linear regression uses all parameters and it’s our baseline

lm1=lm(net~trnd+w_day+mon+hour+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_wdir_10m+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc1_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_TMP_2.m.above.ground+loc3_wdir_10m+loc1_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_TMP_2.m.above.ground+loc4_wdir_10m+loc1_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_TMP_2.m.above.ground+loc5_wdir_10m+loc1_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_TMP_2.m.above.ground+loc6_wdir_10m+loc1_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer+loc7_TMP_2.m.above.ground+loc7_wdir_10m+loc1_ws_10m
       ,all)


summary(lm1)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + loc1_DSWRF_surface + 
##     loc1_RH_2.m.above.ground + loc1_TCDC_low.cloud.layer + loc1_TMP_2.m.above.ground + 
##     loc1_wdir_10m + loc1_ws_10m + loc2_DSWRF_surface + loc2_RH_2.m.above.ground + 
##     loc2_TCDC_low.cloud.layer + loc2_TMP_2.m.above.ground + loc2_wdir_10m + 
##     loc1_ws_10m + loc3_DSWRF_surface + loc3_RH_2.m.above.ground + 
##     loc3_TCDC_low.cloud.layer + loc3_TMP_2.m.above.ground + loc3_wdir_10m + 
##     loc1_ws_10m + loc4_DSWRF_surface + loc4_RH_2.m.above.ground + 
##     loc4_TCDC_low.cloud.layer + loc4_TMP_2.m.above.ground + loc4_wdir_10m + 
##     loc1_ws_10m + loc5_DSWRF_surface + loc5_RH_2.m.above.ground + 
##     loc5_TCDC_low.cloud.layer + loc5_TMP_2.m.above.ground + loc5_wdir_10m + 
##     loc1_ws_10m + loc6_DSWRF_surface + loc6_RH_2.m.above.ground + 
##     loc6_TCDC_low.cloud.layer + loc6_TMP_2.m.above.ground + loc6_wdir_10m + 
##     loc1_ws_10m + loc7_DSWRF_surface + loc7_RH_2.m.above.ground + 
##     loc7_TCDC_low.cloud.layer + loc7_TMP_2.m.above.ground + loc7_wdir_10m + 
##     loc1_ws_10m, data = all)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5672.1  -561.8   -21.7   515.9  8418.0 
## 
## Coefficients:
##                             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                3.061e+02  1.330e+02   2.301 0.021383 *  
## trnd                       2.642e-02  8.691e-04  30.404  < 2e-16 ***
## w_dayCum                   1.138e+02  2.231e+01   5.100 3.42e-07 ***
## w_dayÇar                   1.367e+02  2.235e+01   6.116 9.75e-10 ***
## w_dayPaz                  -2.219e+02  2.233e+01  -9.935  < 2e-16 ***
## w_dayPer                   1.857e+02  2.234e+01   8.313  < 2e-16 ***
## w_dayPts                   1.031e+02  2.235e+01   4.615 3.94e-06 ***
## w_daySal                   1.462e+02  2.238e+01   6.532 6.62e-11 ***
## monAra                     1.578e+02  5.644e+01   2.797 0.005169 ** 
## monEki                    -1.548e+02  3.776e+01  -4.100 4.15e-05 ***
## monEyl                    -3.139e+02  3.138e+01 -10.004  < 2e-16 ***
## monHaz                    -4.143e+01  3.255e+01  -1.273 0.203068    
## monKas                    -3.717e+01  4.716e+01  -0.788 0.430591    
## monMar                     8.045e+01  5.689e+01   1.414 0.157381    
## monMay                    -1.587e+02  3.874e+01  -4.097 4.20e-05 ***
## monNis                    -9.667e+01  5.095e+01  -1.897 0.057811 .  
## monOca                     2.050e+02  6.087e+01   3.368 0.000757 ***
## monŞub                     2.378e+02  6.127e+01   3.881 0.000104 ***
## monTem                     6.148e+01  2.962e+01   2.076 0.037925 *  
## hour1                      1.785e+02  4.115e+01   4.338 1.44e-05 ***
## hour2                      1.338e+02  4.125e+01   3.243 0.001184 ** 
## hour3                      9.590e+01  4.138e+01   2.318 0.020468 *  
## hour4                      4.285e+01  4.158e+01   1.031 0.302735    
## hour5                      3.605e+01  4.176e+01   0.863 0.387913    
## hour6                     -5.076e+01  4.191e+01  -1.211 0.225812    
## hour7                     -2.432e+01  4.193e+01  -0.580 0.561870    
## hour8                      9.537e+01  4.200e+01   2.271 0.023164 *  
## hour9                      2.774e+02  4.268e+01   6.499 8.22e-11 ***
## hour10                     3.014e+02  5.020e+01   6.005 1.94e-09 ***
## hour11                     3.660e+02  5.265e+01   6.951 3.71e-12 ***
## hour12                     1.156e+02  5.490e+01   2.105 0.035289 *  
## hour13                     3.037e+00  5.651e+01   0.054 0.957137    
## hour14                     2.335e+02  5.738e+01   4.070 4.71e-05 ***
## hour15                     2.024e+02  5.745e+01   3.523 0.000428 ***
## hour16                     6.460e+01  5.420e+01   1.192 0.233274    
## hour17                     1.578e+02  5.154e+01   3.061 0.002208 ** 
## hour18                     1.723e+02  4.869e+01   3.538 0.000403 ***
## hour19                     1.999e+02  4.637e+01   4.311 1.63e-05 ***
## hour20                     2.415e+02  4.482e+01   5.389 7.15e-08 ***
## hour21                     3.197e+02  4.369e+01   7.317 2.60e-13 ***
## hour22                     2.126e+02  4.146e+01   5.129 2.94e-07 ***
## hour23                     1.854e+02  4.117e+01   4.503 6.73e-06 ***
## loc1_DSWRF_surface         5.437e-01  1.155e-01   4.706 2.53e-06 ***
## loc1_RH_2.m.above.ground  -3.633e+00  5.952e-01  -6.104 1.05e-09 ***
## loc1_TCDC_low.cloud.layer  2.962e+00  3.452e-01   8.581  < 2e-16 ***
## loc1_TMP_2.m.above.ground -3.208e+01  4.568e+00  -7.023 2.23e-12 ***
## loc1_wdir_10m              1.496e-01  7.659e-02   1.954 0.050764 .  
## loc1_ws_10m               -1.665e+00  4.278e+00  -0.389 0.697177    
## loc2_DSWRF_surface        -7.063e-01  1.221e-01  -5.786 7.29e-09 ***
## loc2_RH_2.m.above.ground  -7.273e-01  4.972e-01  -1.463 0.143503    
## loc2_TCDC_low.cloud.layer  5.290e-01  2.920e-01   1.811 0.070079 .  
## loc2_TMP_2.m.above.ground  8.073e-01  3.855e+00   0.209 0.834117    
## loc2_wdir_10m             -1.918e-01  7.083e-02  -2.708 0.006783 ** 
## loc3_DSWRF_surface         3.007e-01  1.194e-01   2.519 0.011769 *  
## loc3_RH_2.m.above.ground   1.990e-01  8.581e-01   0.232 0.816637    
## loc3_TCDC_low.cloud.layer  2.124e+00  3.265e-01   6.505 7.89e-11 ***
## loc3_TMP_2.m.above.ground  1.043e+01  5.971e+00   1.747 0.080575 .  
## loc3_wdir_10m              2.447e-01  6.793e-02   3.602 0.000317 ***
## loc4_DSWRF_surface        -3.900e-01  1.329e-01  -2.934 0.003353 ** 
## loc4_RH_2.m.above.ground  -6.954e+00  8.225e-01  -8.455  < 2e-16 ***
## loc4_TCDC_low.cloud.layer -8.134e-01  3.249e-01  -2.504 0.012296 *  
## loc4_TMP_2.m.above.ground -7.469e+00  4.516e+00  -1.654 0.098185 .  
## loc4_wdir_10m             -7.713e-01  8.016e-02  -9.622  < 2e-16 ***
## loc5_DSWRF_surface        -1.719e-01  1.273e-01  -1.351 0.176841    
## loc5_RH_2.m.above.ground  -5.195e+00  8.985e-01  -5.782 7.48e-09 ***
## loc5_TCDC_low.cloud.layer  1.713e+00  2.917e-01   5.873 4.34e-09 ***
## loc5_TMP_2.m.above.ground  5.373e-01  6.041e+00   0.089 0.929121    
## loc5_wdir_10m             -1.176e-01  8.556e-02  -1.375 0.169190    
## loc6_DSWRF_surface        -3.349e-01  1.203e-01  -2.785 0.005355 ** 
## loc6_RH_2.m.above.ground  -4.927e+00  8.362e-01  -5.892 3.85e-09 ***
## loc6_TCDC_low.cloud.layer  1.591e+00  3.013e-01   5.280 1.30e-07 ***
## loc6_TMP_2.m.above.ground  2.841e-01  5.839e+00   0.049 0.961188    
## loc6_wdir_10m             -4.353e-01  7.186e-02  -6.058 1.40e-09 ***
## loc7_DSWRF_surface         2.542e-01  1.108e-01   2.296 0.021706 *  
## loc7_RH_2.m.above.ground   5.283e+00  8.485e-01   6.226 4.87e-10 ***
## loc7_TCDC_low.cloud.layer -1.679e-01  2.426e-01  -0.692 0.489042    
## loc7_TMP_2.m.above.ground  4.452e+01  4.753e+00   9.368  < 2e-16 ***
## loc7_wdir_10m              6.462e-01  7.793e-02   8.292  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 971.7 on 26730 degrees of freedom
## Multiple R-squared:  0.1472, Adjusted R-squared:  0.1448 
## F-statistic: 59.94 on 77 and 26730 DF,  p-value: < 2.2e-16
tmp=copy(all)
tmp[,actual:=net]
tmp[,predicted:=predict(lm1,tmp)]
#head(tmp)
ggplot(tmp[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

lm2=lm(net~trnd+w_day+mon+hour,all)
summary(lm2)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour, data = all)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5492.6  -566.8   -28.1   515.0  8524.9 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept) -1.964e+01  4.050e+01  -0.485 0.627711    
## trnd         2.567e-02  8.178e-04  31.390  < 2e-16 ***
## w_dayCum     1.189e+02  2.270e+01   5.237 1.64e-07 ***
## w_dayÇar     1.496e+02  2.270e+01   6.591 4.46e-11 ***
## w_dayPaz    -2.172e+02  2.273e+01  -9.554  < 2e-16 ***
## w_dayPer     1.939e+02  2.270e+01   8.542  < 2e-16 ***
## w_dayPts     1.049e+02  2.274e+01   4.615 3.95e-06 ***
## w_daySal     1.760e+02  2.270e+01   7.751 9.43e-15 ***
## monAra      -3.697e+02  2.982e+01 -12.397  < 2e-16 ***
## monEki      -3.550e+02  2.975e+01 -11.931  < 2e-16 ***
## monEyl      -4.568e+02  2.998e+01 -15.237  < 2e-16 ***
## monHaz      -2.484e+02  3.000e+01  -8.283  < 2e-16 ***
## monKas      -3.635e+02  3.002e+01 -12.108  < 2e-16 ***
## monMar      -3.812e+02  2.988e+01 -12.761  < 2e-16 ***
## monMay      -4.457e+02  2.978e+01 -14.965  < 2e-16 ***
## monNis      -5.232e+02  3.007e+01 -17.399  < 2e-16 ***
## monOca      -4.152e+02  2.837e+01 -14.635  < 2e-16 ***
## monŞub      -3.170e+02  3.063e+01 -10.352  < 2e-16 ***
## monTem      -1.275e+01  2.973e+01  -0.429 0.668041    
## hour1        1.531e+02  4.201e+01   3.643 0.000270 ***
## hour2        8.692e+01  4.201e+01   2.069 0.038570 *  
## hour3        3.182e+01  4.201e+01   0.757 0.448900    
## hour4       -2.721e+01  4.201e+01  -0.648 0.517218    
## hour5       -5.142e+01  4.201e+01  -1.224 0.221019    
## hour6       -1.513e+02  4.201e+01  -3.602 0.000317 ***
## hour7       -1.196e+02  4.201e+01  -2.847 0.004417 ** 
## hour8        5.141e+01  4.201e+01   1.224 0.221141    
## hour9        3.091e+02  4.201e+01   7.358 1.93e-13 ***
## hour10       2.900e+02  4.201e+01   6.902 5.26e-12 ***
## hour11       4.235e+02  4.201e+01  10.081  < 2e-16 ***
## hour12       2.263e+02  4.201e+01   5.386 7.28e-08 ***
## hour13       1.566e+02  4.201e+01   3.726 0.000195 ***
## hour14       4.204e+02  4.201e+01  10.006  < 2e-16 ***
## hour15       4.109e+02  4.201e+01   9.780  < 2e-16 ***
## hour16       3.474e+02  4.201e+01   8.270  < 2e-16 ***
## hour17       4.369e+02  4.201e+01  10.398  < 2e-16 ***
## hour18       4.202e+02  4.201e+01  10.001  < 2e-16 ***
## hour19       3.931e+02  4.201e+01   9.357  < 2e-16 ***
## hour20       3.822e+02  4.201e+01   9.098  < 2e-16 ***
## hour21       4.084e+02  4.201e+01   9.719  < 2e-16 ***
## hour22       2.945e+02  4.201e+01   7.009 2.45e-12 ***
## hour23       2.189e+02  4.201e+01   5.209 1.91e-07 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 992.9 on 26766 degrees of freedom
## Multiple R-squared:  0.1084, Adjusted R-squared:  0.107 
## F-statistic: 79.34 on 41 and 26766 DF,  p-value: < 2.2e-16
tmp2=copy(all)
tmp2[,actual:=net]
tmp2[,predicted:=predict(lm2,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

checkresiduals(lm2$residuals)
## Warning in modeldf.default(object): Could not find appropriate degrees of
## freedom for this model.

tmp2[,prevday:=shift(x=tmp2$net,n=24L,fill=mean(tmp2$net))]
tmp2[,prevweek:=shift(x=tmp2$net,n=168L,fill=mean(tmp2$net))]
tmp2
##                   datetime       net system_direction loc1_DSWRF_surface
##     1: 2019-01-01 00:00:00 -1934.795         Negative               0.00
##     2: 2019-01-01 01:00:00  -750.866         Negative               0.00
##     3: 2019-01-01 02:00:00  -982.767         Negative               0.00
##     4: 2019-01-01 03:00:00  -764.620         Negative               0.00
##     5: 2019-01-01 04:00:00 -1233.289         Negative               0.00
##    ---                                                                  
## 26804: 2022-01-21 19:00:00     0.000                               55.38
## 26805: 2022-01-21 20:00:00     0.000                               44.30
## 26806: 2022-01-21 21:00:00     0.000                               36.92
## 26807: 2022-01-21 22:00:00     0.000                                0.00
## 26808: 2022-01-21 23:00:00     0.000                                0.00
##        loc1_RH_2.m.above.ground loc1_TCDC_low.cloud.layer
##     1:                 84.20000                   0.00000
##     2:                 84.43333                   3.00000
##     3:                 84.66667                   6.00000
##     4:                 84.90000                   9.00000
##     5:                 87.26667                  29.66667
##    ---                                                   
## 26804:                 97.90000                  99.90000
## 26805:                 98.00000                  99.90000
## 26806:                 98.70000                  99.90000
## 26807:                 98.80000                 100.00000
## 26808:                 98.90000                 100.00000
##        loc1_TMP_2.m.above.ground loc1_wdir_10m loc1_ws_10m loc2_DSWRF_surface
##     1:                    -4.081    244.986564    2.302671               0.00
##     2:                    -3.707    245.382871    2.443606               0.00
##     3:                    -3.333    245.779178    2.584541               0.00
##     4:                    -2.959    246.175485    2.725476               0.00
##     5:                    -2.591    240.651199    2.755832               0.00
##    ---                                                                       
## 26804:                    -1.623      2.212407    1.948613              40.46
## 26805:                    -1.285    357.806092    1.767906              32.36
## 26806:                    -0.937    354.741987    2.657864              26.98
## 26807:                    -0.641      7.030764    3.098056               0.00
## 26808:                    -0.540     27.723707    3.520917               0.00
##        loc2_RH_2.m.above.ground loc2_TCDC_low.cloud.layer
##     1:                 59.80000                       0.0
##     2:                 56.93333                       0.0
##     3:                 54.06667                       0.0
##     4:                 51.20000                       0.0
##     5:                 51.56667                       0.0
##    ---                                                   
## 26804:                 66.80000                      99.9
## 26805:                 68.60000                      99.9
## 26806:                 68.50000                      99.9
## 26807:                 73.90000                     100.0
## 26808:                 79.80000                     100.0
##        loc2_TMP_2.m.above.ground loc2_wdir_10m loc2_ws_10m loc3_DSWRF_surface
##     1:                  3.369000      192.5959    3.057220               0.00
##     2:                  3.333000      192.2694    3.202298               0.00
##     3:                  3.297000      191.9429    3.347376               0.00
##     4:                  3.261000      191.6164    3.492454               0.00
##     5:                  3.365667      191.7771    3.608525               0.00
##    ---                                                                       
## 26804:                  4.347000      208.0302    3.265933             163.96
## 26805:                  4.295000      203.6774    3.355885             131.16
## 26806:                  4.533000      206.7301    3.698566             109.30
## 26807:                  4.649000      211.2813    4.276995               0.00
## 26808:                  4.530000      212.4927    3.987479               0.00
##        loc3_RH_2.m.above.ground loc3_TCDC_low.cloud.layer
##     1:                 84.40000                 1.0000000
##     2:                 85.06667                 0.6666667
##     3:                 85.73333                 0.3333333
##     4:                 86.40000                 0.0000000
##     5:                 87.83333                 1.3333333
##    ---                                                   
## 26804:                 72.40000                 0.0000000
## 26805:                 71.90000                 0.0000000
## 26806:                 72.70000                 0.0000000
## 26807:                 74.60000                 5.0000000
## 26808:                 77.70000                 4.8000000
##        loc3_TMP_2.m.above.ground loc3_wdir_10m loc3_ws_10m loc4_DSWRF_surface
##     1:                 -6.211000     351.73126    1.784915               0.00
##     2:                 -5.950333     350.11411    1.928927               0.00
##     3:                 -5.689667     348.49697    2.072939               0.00
##     4:                 -5.429000     346.87983    2.216951               0.00
##     5:                 -5.094333     337.72332    1.997965               0.00
##    ---                                                                       
## 26804:                 -2.723000      43.76432    3.014659              30.56
## 26805:                 -3.055000      45.90085    2.955315              24.46
## 26806:                 -3.107000      42.20943    2.763192              20.36
## 26807:                 -3.091000      39.53944    2.716282               0.00
## 26808:                 -2.910000      51.42887    2.849656               0.00
##        loc4_RH_2.m.above.ground loc4_TCDC_low.cloud.layer
##     1:                 52.20000                   0.00000
##     2:                 53.90000                   9.00000
##     3:                 55.60000                  18.00000
##     4:                 57.30000                  27.00000
##     5:                 59.76667                  51.33333
##    ---                                                   
## 26804:                 88.20000                  98.10000
## 26805:                 89.20000                  98.50000
## 26806:                 89.20000                  98.70000
## 26807:                 89.30000                 100.00000
## 26808:                 90.10000                 100.00000
##        loc4_TMP_2.m.above.ground loc4_wdir_10m loc4_ws_10m loc5_DSWRF_surface
##     1:                  7.309000      248.9115    5.151732               0.00
##     2:                  7.203000      248.8854    5.512422               0.00
##     3:                  7.097000      248.8594    5.873112               0.00
##     4:                  6.991000      248.8333    6.233802               0.00
##     5:                  6.902333      248.9605    6.574972               0.00
##    ---                                                                       
## 26804:                  9.287000      352.6565    4.262119             116.34
## 26805:                  9.265000      352.4482    4.243415              93.08
## 26806:                  9.233000      350.1066    4.036709              77.56
## 26807:                  9.189000      346.2419    3.577402               0.00
## 26808:                  9.140000      339.1743    3.634154               0.00
##        loc5_RH_2.m.above.ground loc5_TCDC_low.cloud.layer
##     1:                 88.30000                 0.0000000
##     2:                 87.66667                 0.0000000
##     3:                 87.03333                 0.0000000
##     4:                 86.40000                 0.0000000
##     5:                 86.66667                 0.3333333
##    ---                                                   
## 26804:                 84.30000                69.8000000
## 26805:                 81.30000                69.5000000
## 26806:                 86.40000                70.2000000
## 26807:                 89.50000               100.0000000
## 26808:                 94.00000                99.1000000
##        loc5_TMP_2.m.above.ground loc5_wdir_10m loc5_ws_10m loc6_DSWRF_surface
##     1:                 -4.191000     297.65206    3.913716               0.00
##     2:                 -4.070333     295.48408    4.243639               0.00
##     3:                 -3.949667     293.31610    4.573562               0.00
##     4:                 -3.829000     291.14811    4.903485               0.00
##     5:                 -3.864333     290.61532    4.996934               0.00
##    ---                                                                       
## 26804:                  1.997000      19.05118    4.366314             114.36
## 26805:                  2.185000      29.69346    5.556206              91.48
## 26806:                  1.463000      47.68443    4.525578              76.24
## 26807:                  1.679000      63.83293    3.843080               0.00
## 26808:                  1.490000      56.89650    3.948929               0.00
##        loc6_RH_2.m.above.ground loc6_TCDC_low.cloud.layer
##     1:                 92.10000                  28.00000
##     2:                 92.50000                  25.66667
##     3:                 92.90000                  23.33333
##     4:                 93.30000                  21.00000
##     5:                 94.03333                  21.00000
##    ---                                                   
## 26804:                 73.20000                  74.30000
## 26805:                 72.20000                  79.40000
## 26806:                 74.20000                  82.60000
## 26807:                 89.20000                 100.00000
## 26808:                 82.50000                 100.00000
##        loc6_TMP_2.m.above.ground loc6_wdir_10m loc6_ws_10m loc7_DSWRF_surface
##     1:                 -7.041000      280.4415    1.745607               0.00
##     2:                 -7.200333      274.3150    1.962468               0.00
##     3:                 -7.359667      268.1885    2.179329               0.00
##     4:                 -7.519000      262.0619    2.396190               0.00
##     5:                 -7.527667      260.3688    2.522906               0.00
##    ---                                                                       
## 26804:                 -1.953000      324.6584    2.117349               7.32
## 26805:                 -1.895000      312.2008    2.332197               5.86
## 26806:                 -1.907000      331.4124    2.160014               4.88
## 26807:                 -2.371000      300.8462    2.388677               0.00
## 26808:                 -1.210000      345.0344    2.563677               0.00
##        loc7_RH_2.m.above.ground loc7_TCDC_low.cloud.layer
##     1:                 65.80000                         0
##     2:                 66.40000                         0
##     3:                 67.00000                         0
##     4:                 67.60000                         0
##     5:                 67.83333                         0
##    ---                                                   
## 26804:                 94.90000                       100
## 26805:                 94.80000                       100
## 26806:                 95.00000                       100
## 26807:                 95.20000                       100
## 26808:                 95.20000                       100
##        loc7_TMP_2.m.above.ground loc7_wdir_10m loc7_ws_10m  trnd w_day mon hour
##     1:                  7.239000      250.5319    5.861836     1   Sal Oca    0
##     2:                  7.153000      250.9196    6.254371     2   Sal Oca    1
##     3:                  7.067000      251.3073    6.646906     3   Sal Oca    2
##     4:                  6.981000      251.6950    7.039441     4   Sal Oca    3
##     5:                  6.962333      252.8526    7.291198     5   Sal Oca    4
##    ---                                                                         
## 26804:                  3.577000      192.1873    6.417472 26804   Cum Oca   19
## 26805:                  2.815000      181.7018    6.656326 26805   Cum Oca   20
## 26806:                  2.743000      185.1566    6.047797 26806   Cum Oca   21
## 26807:                  2.399000      184.1567    6.081236 26807   Cum Oca   22
## 26808:                  2.180000      182.7795    6.641093 26808   Cum Oca   23
##              Date    actual predicted   prevday  prevweek
##     1: 2019-01-01 -1934.795 -258.8107  287.9239  287.9239
##     2: 2019-01-01  -750.866 -105.7097  287.9239  287.9239
##     3: 2019-01-01  -982.767 -171.8369  287.9239  287.9239
##     4: 2019-01-01  -764.620 -226.9174  287.9239  287.9239
##     5: 2019-01-01 -1233.289 -285.9188  287.9239  287.9239
##    ---                                                   
## 26804: 2022-01-21     0.000  765.3269 2043.4290 -425.3450
## 26805: 2022-01-21     0.000  754.4451 2171.4590 -339.5620
## 26806: 2022-01-21     0.000  780.6004 2981.0760 -356.4620
## 26807: 2022-01-21     0.000  666.7590 2627.5980 -522.6550
## 26808: 2022-01-21     0.000  591.1566    0.0000  147.8350

After that try 13 linear regression to reach better results

lm2_1=lm(net~trnd+w_day+mon+hour+prevday+prevweek,
                 tmp2)
summary(lm2_1)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek, 
##     data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5876.5  -521.5   -10.9   500.2  6631.5 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept) -7.709e+01  3.656e+01  -2.108 0.035014 *  
## trnd         1.308e-02  7.617e-04  17.173  < 2e-16 ***
## w_dayCum     7.933e+01  2.050e+01   3.870 0.000109 ***
## w_dayÇar     1.143e+02  2.051e+01   5.575 2.50e-08 ***
## w_dayPaz    -1.510e+02  2.056e+01  -7.346 2.10e-13 ***
## w_dayPer     1.679e+02  2.052e+01   8.182 2.93e-16 ***
## w_dayPts     2.393e+02  2.062e+01  11.606  < 2e-16 ***
## w_daySal     1.678e+02  2.051e+01   8.180 2.95e-16 ***
## monAra      -1.854e+02  2.705e+01  -6.856 7.26e-12 ***
## monEki      -1.600e+02  2.702e+01  -5.922 3.21e-09 ***
## monEyl      -2.305e+02  2.724e+01  -8.460  < 2e-16 ***
## monHaz      -8.766e+01  2.722e+01  -3.220 0.001283 ** 
## monKas      -1.667e+02  2.727e+01  -6.113 9.90e-10 ***
## monMar      -1.850e+02  2.713e+01  -6.817 9.51e-12 ***
## monMay      -1.908e+02  2.717e+01  -7.021 2.25e-12 ***
## monNis      -2.542e+02  2.743e+01  -9.269  < 2e-16 ***
## monOca      -1.894e+02  2.583e+01  -7.330 2.36e-13 ***
## monŞub      -1.405e+02  2.779e+01  -5.057 4.29e-07 ***
## monTem       7.102e+00  2.684e+01   0.265 0.791309    
## hour1        7.730e+01  3.794e+01   2.038 0.041600 *  
## hour2        4.389e+01  3.793e+01   1.157 0.247137    
## hour3        1.599e+01  3.792e+01   0.422 0.673182    
## hour4       -1.380e+01  3.792e+01  -0.364 0.715899    
## hour5       -2.599e+01  3.792e+01  -0.685 0.493056    
## hour6       -7.628e+01  3.794e+01  -2.011 0.044361 *  
## hour7       -6.048e+01  3.793e+01  -1.594 0.110863    
## hour8        2.604e+01  3.792e+01   0.687 0.492355    
## hour9        1.562e+02  3.799e+01   4.112 3.94e-05 ***
## hour10       1.464e+02  3.798e+01   3.854 0.000116 ***
## hour11       2.139e+02  3.805e+01   5.621 1.91e-08 ***
## hour12       1.143e+02  3.796e+01   3.012 0.002598 ** 
## hour13       7.913e+01  3.794e+01   2.086 0.037009 *  
## hour14       2.123e+02  3.805e+01   5.580 2.43e-08 ***
## hour15       2.076e+02  3.804e+01   5.456 4.92e-08 ***
## hour16       1.754e+02  3.801e+01   4.615 3.95e-06 ***
## hour17       2.205e+02  3.806e+01   5.793 7.01e-09 ***
## hour18       2.122e+02  3.805e+01   5.577 2.47e-08 ***
## hour19       1.985e+02  3.803e+01   5.220 1.80e-07 ***
## hour20       1.930e+02  3.803e+01   5.076 3.88e-07 ***
## hour21       2.063e+02  3.804e+01   5.423 5.90e-08 ***
## hour22       1.490e+02  3.798e+01   3.923 8.76e-05 ***
## hour23       1.108e+02  3.796e+01   2.919 0.003514 ** 
## prevday      4.179e-01  5.542e-03  75.405  < 2e-16 ***
## prevweek     7.769e-02  5.545e-03  14.011  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 896.2 on 26764 degrees of freedom
## Multiple R-squared:  0.2737, Adjusted R-squared:  0.2726 
## F-statistic: 234.6 on 43 and 26764 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_1,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~w_day+mon+hour+prevday+prevweek+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_wdir_10m+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_TMP_2.m.above.ground+loc3_wdir_10m+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_TMP_2.m.above.ground+loc4_wdir_10m+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_TMP_2.m.above.ground+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_TMP_2.m.above.ground+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer+loc7_TMP_2.m.above.ground+loc7_wdir_10m+loc7_ws_10m,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc7tmp2ag:=as.numeric(loc7_TMP_2.m.above.ground>=30)]
tmp2[,pv:=as.numeric(prevweek>=2070)]
tmp2[,loc7tmp2ag2:=as.numeric(loc7_TMP_2.m.above.ground>=31)]

lm2_2=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2,
                 tmp2)
summary(lm2_2)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5545.6  -516.0    -7.5   499.2  5789.6 
## 
## Coefficients:
##                             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)               -1.136e+02  3.596e+01  -3.159 0.001583 ** 
## trnd                       1.324e-02  7.488e-04  17.679  < 2e-16 ***
## w_dayCum                   7.455e+01  2.015e+01   3.699 0.000217 ***
## w_dayÇar                   1.103e+02  2.016e+01   5.470 4.55e-08 ***
## w_dayPaz                  -1.566e+02  2.021e+01  -7.746 9.82e-15 ***
## w_dayPer                   1.563e+02  2.017e+01   7.746 9.80e-15 ***
## w_dayPts                   2.225e+02  2.028e+01  10.975  < 2e-16 ***
## w_daySal                   1.651e+02  2.016e+01   8.186 2.82e-16 ***
## monAra                    -1.331e+02  2.664e+01  -4.994 5.94e-07 ***
## monEki                    -1.102e+02  2.661e+01  -4.140 3.48e-05 ***
## monEyl                    -1.827e+02  2.682e+01  -6.813 9.75e-12 ***
## monHaz                    -3.834e+01  2.681e+01  -1.430 0.152707    
## monKas                    -1.165e+02  2.686e+01  -4.339 1.43e-05 ***
## monMar                    -1.394e+02  2.671e+01  -5.217 1.83e-07 ***
## monMay                    -1.485e+02  2.674e+01  -5.553 2.84e-08 ***
## monNis                    -2.128e+02  2.700e+01  -7.884 3.29e-15 ***
## monOca                    -1.443e+02  2.544e+01  -5.673 1.42e-08 ***
## monŞub                    -9.414e+01  2.736e+01  -3.441 0.000580 ***
## monTem                     6.196e+01  2.644e+01   2.343 0.019137 *  
## hour1                      8.299e+01  3.729e+01   2.225 0.026077 *  
## hour2                      4.712e+01  3.728e+01   1.264 0.206257    
## hour3                      1.719e+01  3.728e+01   0.461 0.644731    
## hour4                     -1.480e+01  3.728e+01  -0.397 0.691351    
## hour5                     -2.790e+01  3.728e+01  -0.748 0.454281    
## hour6                     -8.191e+01  3.729e+01  -2.196 0.028067 *  
## hour7                     -6.490e+01  3.729e+01  -1.741 0.081767 .  
## hour8                      2.794e+01  3.728e+01   0.750 0.453555    
## hour9                      1.677e+02  3.735e+01   4.490 7.16e-06 ***
## hour10                     1.572e+02  3.734e+01   4.209 2.57e-05 ***
## hour11                     2.209e+02  3.741e+01   5.906 3.55e-09 ***
## hour12                     1.140e+02  3.731e+01   3.055 0.002250 ** 
## hour13                     7.622e+01  3.729e+01   2.044 0.040989 *  
## hour14                     2.105e+02  3.740e+01   5.627 1.85e-08 ***
## hour15                     1.966e+02  3.740e+01   5.258 1.47e-07 ***
## hour16                     1.665e+02  3.737e+01   4.457 8.36e-06 ***
## hour17                     2.193e+02  3.742e+01   5.861 4.66e-09 ***
## hour18                     2.191e+02  3.740e+01   5.858 4.75e-09 ***
## hour19                     2.088e+02  3.739e+01   5.584 2.37e-08 ***
## hour20                     1.985e+02  3.738e+01   5.310 1.10e-07 ***
## hour21                     2.128e+02  3.740e+01   5.689 1.29e-08 ***
## hour22                     1.599e+02  3.734e+01   4.283 1.85e-05 ***
## hour23                     1.189e+02  3.731e+01   3.187 0.001441 ** 
## prevday                    3.916e-01  5.515e-03  71.004  < 2e-16 ***
## prevweek                   6.677e-02  5.463e-03  12.222  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2  4.872e+03  1.594e+02  30.561  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 880.9 on 26763 degrees of freedom
## Multiple R-squared:  0.2982, Adjusted R-squared:  0.2971 
## F-statistic: 258.5 on 44 and 26763 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_2,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~w_day+mon+hour+prevday+prevweek+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_wdir_10m+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_TMP_2.m.above.ground+loc3_wdir_10m+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_TMP_2.m.above.ground+loc4_wdir_10m+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_TMP_2.m.above.ground+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_TMP_2.m.above.ground+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer+loc7_wdir_10m+loc7_ws_10m,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc5tmp2ag:=as.numeric(loc5_TMP_2.m.above.ground>=29)]
tmp2[,pd:=as.numeric(prevday>=2329)]
tmp2[,workday:=as.numeric(!w_day %in% c('Cmt','Paz'))]

lm2_3=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday,
                 tmp2)
summary(lm2_3)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5168.1  -512.6    -6.8   497.3  5811.9 
## 
## Coefficients:
##                             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)               -1.367e+02  3.561e+01  -3.838 0.000124 ***
## trnd                       1.325e-02  7.411e-04  17.885  < 2e-16 ***
## w_dayCum                   6.758e+01  1.995e+01   3.388 0.000705 ***
## w_dayÇar                   1.093e+02  1.995e+01   5.480 4.30e-08 ***
## w_dayPaz                  -1.485e+02  2.001e+01  -7.421 1.20e-13 ***
## w_dayPer                   1.542e+02  1.996e+01   7.724 1.16e-14 ***
## w_dayPts                   2.239e+02  2.007e+01  11.160  < 2e-16 ***
## w_daySal                   1.685e+02  1.996e+01   8.442  < 2e-16 ***
## monAra                    -9.618e+01  2.641e+01  -3.641 0.000272 ***
## monEki                    -7.438e+01  2.638e+01  -2.819 0.004814 ** 
## monEyl                    -1.691e+02  2.655e+01  -6.369 1.93e-10 ***
## monHaz                    -1.134e+01  2.656e+01  -0.427 0.669421    
## monKas                    -8.077e+01  2.662e+01  -3.034 0.002417 ** 
## monMar                    -1.087e+02  2.647e+01  -4.105 4.06e-05 ***
## monMay                    -1.200e+02  2.650e+01  -4.529 5.96e-06 ***
## monNis                    -1.864e+02  2.674e+01  -6.972 3.20e-12 ***
## monOca                    -1.136e+02  2.521e+01  -4.506 6.62e-06 ***
## monŞub                    -6.207e+01  2.711e+01  -2.290 0.022055 *  
## monTem                     6.308e+01  2.617e+01   2.410 0.015956 *  
## hour1                      8.814e+01  3.691e+01   2.388 0.016953 *  
## hour2                      5.005e+01  3.690e+01   1.356 0.174978    
## hour3                      1.826e+01  3.689e+01   0.495 0.620646    
## hour4                     -1.571e+01  3.689e+01  -0.426 0.670167    
## hour5                     -2.962e+01  3.689e+01  -0.803 0.422043    
## hour6                     -8.700e+01  3.691e+01  -2.357 0.018429 *  
## hour7                     -6.892e+01  3.690e+01  -1.868 0.061823 .  
## hour8                      2.968e+01  3.689e+01   0.804 0.421157    
## hour9                      1.781e+02  3.697e+01   4.818 1.46e-06 ***
## hour10                     1.654e+02  3.696e+01   4.474 7.70e-06 ***
## hour11                     2.208e+02  3.702e+01   5.965 2.47e-09 ***
## hour12                     1.057e+02  3.693e+01   2.863 0.004203 ** 
## hour13                     5.769e+01  3.692e+01   1.563 0.118124    
## hour14                     1.865e+02  3.703e+01   5.036 4.78e-07 ***
## hour15                     1.738e+02  3.703e+01   4.694 2.69e-06 ***
## hour16                     1.440e+02  3.699e+01   3.892 9.96e-05 ***
## hour17                     1.943e+02  3.705e+01   5.244 1.58e-07 ***
## hour18                     2.142e+02  3.702e+01   5.786 7.30e-09 ***
## hour19                     2.070e+02  3.701e+01   5.593 2.25e-08 ***
## hour20                     2.129e+02  3.700e+01   5.753 8.88e-09 ***
## hour21                     2.280e+02  3.702e+01   6.159 7.44e-10 ***
## hour22                     1.699e+02  3.696e+01   4.596 4.32e-06 ***
## hour23                     1.263e+02  3.693e+01   3.420 0.000627 ***
## prevday                    3.562e-01  5.660e-03  62.929  < 2e-16 ***
## prevweek                   6.855e-02  5.407e-03  12.677  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2  4.055e+03  1.615e+02  25.109  < 2e-16 ***
## loc5tmp2ag:pd:workday      1.764e+03  7.454e+01  23.666  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 871.9 on 26762 degrees of freedom
## Multiple R-squared:  0.3126, Adjusted R-squared:  0.3115 
## F-statistic: 270.5 on 45 and 26762 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_3,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~mon+hour+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_wdir_10m+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_TMP_2.m.above.ground+loc3_wdir_10m+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_TMP_2.m.above.ground+loc4_wdir_10m+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_TMP_2.m.above.ground+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer+loc7_wdir_10m+loc7_ws_10m,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc4tmp2ag:=as.numeric(loc4_TMP_2.m.above.ground<8.6)]
tmp2[,whour:=as.numeric(as.numeric(hour)<9 | as.numeric(hour)>18)]
tmp2[,loc3wdir:=as.numeric(loc3_wdir_10m<336)]

lm2_4=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir,
                 tmp2)
summary(lm2_4)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir, 
##     data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5167.8  -508.3    -8.1   496.9  5795.6 
## 
## Coefficients:
##                             Estimate Std. Error t value Pr(>|t|)    
## (Intercept)               -1.459e+02  3.559e+01  -4.099 4.16e-05 ***
## trnd                       1.318e-02  7.404e-04  17.805  < 2e-16 ***
## w_dayCum                   6.535e+01  1.993e+01   3.279 0.001043 ** 
## w_dayÇar                   1.099e+02  1.993e+01   5.514 3.54e-08 ***
## w_dayPaz                  -1.487e+02  1.999e+01  -7.438 1.05e-13 ***
## w_dayPer                   1.529e+02  1.995e+01   7.667 1.83e-14 ***
## w_dayPts                   2.244e+02  2.005e+01  11.195  < 2e-16 ***
## w_daySal                   1.711e+02  1.994e+01   8.581  < 2e-16 ***
## monAra                    -1.263e+02  2.671e+01  -4.727 2.29e-06 ***
## monEki                    -7.533e+01  2.635e+01  -2.859 0.004259 ** 
## monEyl                    -1.702e+02  2.653e+01  -6.415 1.43e-10 ***
## monHaz                    -1.262e+01  2.653e+01  -0.476 0.634319    
## monKas                    -9.099e+01  2.663e+01  -3.416 0.000635 ***
## monMar                    -1.408e+02  2.681e+01  -5.253 1.51e-07 ***
## monMay                    -1.217e+02  2.647e+01  -4.596 4.32e-06 ***
## monNis                    -2.009e+02  2.679e+01  -7.499 6.65e-14 ***
## monOca                    -1.625e+02  2.606e+01  -6.235 4.60e-10 ***
## monŞub                    -1.059e+02  2.774e+01  -3.816 0.000136 ***
## monTem                     6.290e+01  2.615e+01   2.406 0.016148 *  
## hour1                      8.743e+01  3.688e+01   2.371 0.017755 *  
## hour2                      4.877e+01  3.686e+01   1.323 0.185807    
## hour3                      1.483e+01  3.686e+01   0.402 0.687405    
## hour4                     -1.996e+01  3.686e+01  -0.541 0.588249    
## hour5                     -3.595e+01  3.687e+01  -0.975 0.329594    
## hour6                     -9.451e+01  3.689e+01  -2.562 0.010416 *  
## hour7                     -7.770e+01  3.689e+01  -2.106 0.035180 *  
## hour8                      5.609e+01  3.704e+01   1.514 0.129920    
## hour9                      2.051e+02  3.712e+01   5.526 3.30e-08 ***
## hour10                     1.923e+02  3.711e+01   5.184 2.19e-07 ***
## hour11                     2.482e+02  3.718e+01   6.677 2.49e-11 ***
## hour12                     1.326e+02  3.708e+01   3.577 0.000349 ***
## hour13                     8.445e+01  3.706e+01   2.278 0.022706 *  
## hour14                     2.140e+02  3.719e+01   5.753 8.84e-09 ***
## hour15                     2.012e+02  3.718e+01   5.412 6.29e-08 ***
## hour16                     1.712e+02  3.715e+01   4.610 4.05e-06 ***
## hour17                     2.218e+02  3.720e+01   5.961 2.53e-09 ***
## hour18                     2.312e+02  3.706e+01   6.238 4.49e-10 ***
## hour19                     2.189e+02  3.701e+01   5.915 3.36e-09 ***
## hour20                     2.212e+02  3.698e+01   5.980 2.25e-09 ***
## hour21                     2.345e+02  3.699e+01   6.338 2.36e-10 ***
## hour22                     1.743e+02  3.693e+01   4.721 2.36e-06 ***
## hour23                     1.280e+02  3.689e+01   3.470 0.000521 ***
## prevday                    3.548e-01  5.658e-03  62.712  < 2e-16 ***
## prevweek                   6.753e-02  5.404e-03  12.498  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2  4.058e+03  1.613e+02  25.150  < 2e-16 ***
## loc5tmp2ag:pd:workday      1.757e+03  7.447e+01  23.594  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir  1.506e+02  2.065e+01   7.292 3.13e-13 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 871 on 26761 degrees of freedom
## Multiple R-squared:  0.314,  Adjusted R-squared:  0.3128 
## F-statistic: 266.3 on 46 and 26761 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_4,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~mon+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_wdir_10m+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_TMP_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_wdir_10m+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_TMP_2.m.above.ground+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer+loc7_wdir_10m+loc7_ws_10m,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc6tmp2ag:=as.numeric(loc6_TMP_2.m.above.ground>=26)]
tmp2[,loc7ws:=as.numeric(loc7_ws_10m<3)]
tmp2[,loc1wdir:=as.numeric(loc1_wdir_10m>=172)]

lm2_5=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir,
                 tmp2)
summary(lm2_5)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -5121.1  -509.0    -7.6   497.5  5052.3 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -1.542e+02  3.545e+01  -4.350 1.37e-05 ***
## trnd                        1.295e-02  7.375e-04  17.553  < 2e-16 ***
## w_dayCum                    6.639e+01  1.985e+01   3.345 0.000825 ***
## w_dayÇar                    1.115e+02  1.985e+01   5.615 1.98e-08 ***
## w_dayPaz                   -1.499e+02  1.991e+01  -7.532 5.17e-14 ***
## w_dayPer                    1.538e+02  1.986e+01   7.744 9.99e-15 ***
## w_dayPts                    2.194e+02  1.997e+01  10.985  < 2e-16 ***
## w_daySal                    1.679e+02  1.986e+01   8.455  < 2e-16 ***
## monAra                     -1.079e+02  2.663e+01  -4.051 5.12e-05 ***
## monEki                     -6.422e+01  2.626e+01  -2.446 0.014459 *  
## monEyl                     -1.528e+02  2.644e+01  -5.779 7.60e-09 ***
## monHaz                      1.097e+00  2.644e+01   0.041 0.966911    
## monKas                     -7.347e+01  2.655e+01  -2.767 0.005661 ** 
## monMar                     -1.246e+02  2.672e+01  -4.662 3.14e-06 ***
## monMay                     -1.096e+02  2.637e+01  -4.156 3.25e-05 ***
## monNis                     -1.855e+02  2.670e+01  -6.948 3.78e-12 ***
## monOca                     -1.455e+02  2.598e+01  -5.601 2.15e-08 ***
## monŞub                     -8.946e+01  2.765e+01  -3.236 0.001215 ** 
## monTem                      6.803e+01  2.604e+01   2.612 0.009002 ** 
## hour1                       8.800e+01  3.672e+01   2.396 0.016572 *  
## hour2                       4.913e+01  3.671e+01   1.338 0.180857    
## hour3                       1.505e+01  3.671e+01   0.410 0.681823    
## hour4                      -1.993e+01  3.671e+01  -0.543 0.587265    
## hour5                      -3.594e+01  3.672e+01  -0.979 0.327656    
## hour6                      -9.483e+01  3.674e+01  -2.581 0.009850 ** 
## hour7                      -7.787e+01  3.674e+01  -2.120 0.034045 *  
## hour8                       5.550e+01  3.688e+01   1.505 0.132447    
## hour9                       2.029e+02  3.696e+01   5.489 4.09e-08 ***
## hour10                      1.812e+02  3.696e+01   4.903 9.49e-07 ***
## hour11                      2.394e+02  3.703e+01   6.465 1.03e-10 ***
## hour12                      1.143e+02  3.695e+01   3.094 0.001980 ** 
## hour13                      6.783e+01  3.693e+01   1.837 0.066272 .  
## hour14                      2.028e+02  3.704e+01   5.475 4.42e-08 ***
## hour15                      1.971e+02  3.703e+01   5.322 1.03e-07 ***
## hour16                      1.676e+02  3.699e+01   4.529 5.94e-06 ***
## hour17                      2.198e+02  3.705e+01   5.932 3.04e-09 ***
## hour18                      2.320e+02  3.691e+01   6.287 3.29e-10 ***
## hour19                      2.203e+02  3.685e+01   5.979 2.28e-09 ***
## hour20                      2.230e+02  3.683e+01   6.054 1.43e-09 ***
## hour21                      2.364e+02  3.684e+01   6.417 1.41e-10 ***
## hour22                      1.740e+02  3.678e+01   4.731 2.24e-06 ***
## hour23                      1.288e+02  3.674e+01   3.505 0.000458 ***
## prevday                     3.520e-01  5.638e-03  62.441  < 2e-16 ***
## prevweek                    6.678e-02  5.382e-03  12.409  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.692e+03  1.626e+02  22.712  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.609e+03  7.483e+01  21.506  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   1.461e+02  2.057e+01   7.105 1.23e-12 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.428e+03  9.591e+01  14.889  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 867.5 on 26760 degrees of freedom
## Multiple R-squared:  0.3196, Adjusted R-squared:  0.3184 
## F-statistic: 267.5 on 47 and 26760 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_5,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~mon+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_TMP_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_wdir_10m+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer+loc7_wdir_10m,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc4wdir:=as.numeric(loc4_wdir_10m<170)]
tmp2[,loc7wdir:=as.numeric(loc7_wdir_10m>=214)]
tmp2[,loc3tmp2ag:=as.numeric(loc3_TMP_2.m.above.ground>=20)]

lm2_6=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag,
                 tmp2)
summary(lm2_6)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag, 
##     data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4970.8  -512.5    -4.6   497.1  5010.2 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.212e+02  3.670e+01  -8.751  < 2e-16 ***
## trnd                        1.380e-02  7.335e-04  18.808  < 2e-16 ***
## w_dayCum                    6.751e+01  1.970e+01   3.427 0.000611 ***
## w_dayÇar                    1.033e+02  1.970e+01   5.245 1.58e-07 ***
## w_dayPaz                   -1.541e+02  1.976e+01  -7.799 6.48e-15 ***
## w_dayPer                    1.520e+02  1.971e+01   7.709 1.31e-14 ***
## w_dayPts                    2.138e+02  1.982e+01  10.784  < 2e-16 ***
## w_daySal                    1.556e+02  1.972e+01   7.889 3.16e-15 ***
## monAra                      7.151e+01  2.923e+01   2.446 0.014448 *  
## monEki                      4.220e+01  2.758e+01   1.530 0.125971    
## monEyl                     -1.025e+02  2.665e+01  -3.848 0.000119 ***
## monHaz                      6.364e+01  2.652e+01   2.399 0.016428 *  
## monKas                      9.300e+01  2.897e+01   3.210 0.001330 ** 
## monMar                      5.181e+01  2.936e+01   1.765 0.077643 .  
## monMay                     -3.200e+01  2.695e+01  -1.188 0.234986    
## monNis                     -3.963e+01  2.888e+01  -1.372 0.170055    
## monOca                      4.075e+01  2.882e+01   1.414 0.157310    
## monŞub                      9.726e+01  3.032e+01   3.207 0.001342 ** 
## monTem                      5.717e+01  2.585e+01   2.211 0.027026 *  
## hour1                       1.010e+02  3.645e+01   2.770 0.005608 ** 
## hour2                       6.896e+01  3.645e+01   1.892 0.058505 .  
## hour3                       4.178e+01  3.646e+01   1.146 0.251839    
## hour4                       1.075e+01  3.647e+01   0.295 0.768287    
## hour5                       5.069e+00  3.650e+01   0.139 0.889560    
## hour6                      -5.554e+01  3.652e+01  -1.521 0.128338    
## hour7                      -4.368e+01  3.649e+01  -1.197 0.231396    
## hour8                       6.874e+01  3.663e+01   1.877 0.060570 .  
## hour9                       2.000e+02  3.674e+01   5.443 5.30e-08 ***
## hour10                      1.556e+02  3.681e+01   4.228 2.37e-05 ***
## hour11                      1.999e+02  3.695e+01   5.410 6.35e-08 ***
## hour12                      5.801e+01  3.697e+01   1.569 0.116644    
## hour13                      4.936e-01  3.703e+01   0.013 0.989366    
## hour14                      1.370e+02  3.715e+01   3.687 0.000227 ***
## hour15                      1.278e+02  3.715e+01   3.441 0.000581 ***
## hour16                      9.723e+01  3.711e+01   2.620 0.008802 ** 
## hour17                      1.458e+02  3.715e+01   3.924 8.74e-05 ***
## hour18                      1.592e+02  3.691e+01   4.313 1.61e-05 ***
## hour19                      1.490e+02  3.679e+01   4.050 5.13e-05 ***
## hour20                      1.648e+02  3.667e+01   4.495 6.99e-06 ***
## hour21                      1.962e+02  3.661e+01   5.358 8.46e-08 ***
## hour22                      1.465e+02  3.652e+01   4.011 6.06e-05 ***
## hour23                      1.204e+02  3.646e+01   3.303 0.000958 ***
## prevday                     3.400e-01  5.641e-03  60.270  < 2e-16 ***
## prevweek                    6.358e-02  5.345e-03  11.896  < 2e-16 ***
## loc3tmp2ag                  2.022e+02  1.846e+01  10.959  < 2e-16 ***
## loc4wdir:loc7wdir           2.809e+02  1.714e+01  16.386  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.702e+03  1.613e+02  22.945  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.570e+03  7.428e+01  21.133  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   1.209e+02  2.061e+01   5.869 4.45e-09 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.409e+03  9.525e+01  14.798  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 860.8 on 26758 degrees of freedom
## Multiple R-squared:   0.33,  Adjusted R-squared:  0.3288 
## F-statistic:   269 on 49 and 26758 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_6,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~mon+
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_TCDC_low.cloud.layer+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_RH_2.m.above.ground+loc4_TCDC_low.cloud.layer+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc3tcdc:=as.numeric(loc3_TCDC_low.cloud.layer>=8.6)]
tmp2[,mon1:=as.numeric(!mon %in% c('Haz','Tem','Ağu','Eyl','Eki'))]
tmp2[,loc4rh:=as.numeric(loc4_RH_2.m.above.ground<69)]

lm2_7=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh,
                 tmp2)
summary(lm2_7)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4954.3  -507.6    -3.8   493.2  4819.8 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.348e+02  3.635e+01  -9.209  < 2e-16 ***
## trnd                        1.427e-02  7.267e-04  19.638  < 2e-16 ***
## w_dayCum                    5.539e+01  1.951e+01   2.838 0.004536 ** 
## w_dayÇar                    9.369e+01  1.952e+01   4.800 1.59e-06 ***
## w_dayPaz                   -1.543e+02  1.957e+01  -7.889 3.17e-15 ***
## w_dayPer                    1.405e+02  1.953e+01   7.195 6.41e-13 ***
## w_dayPts                    2.160e+02  1.963e+01  11.004  < 2e-16 ***
## w_daySal                    1.555e+02  1.953e+01   7.964 1.73e-15 ***
## monAra                      6.120e+01  2.895e+01   2.114 0.034541 *  
## monEki                      7.554e+01  2.735e+01   2.762 0.005746 ** 
## monEyl                     -8.632e+01  2.640e+01  -3.270 0.001077 ** 
## monHaz                      7.978e+01  2.628e+01   3.036 0.002397 ** 
## monKas                      8.858e+01  2.869e+01   3.087 0.002021 ** 
## monMar                     -1.217e+00  2.917e+01  -0.042 0.966721    
## monMay                     -6.705e+01  2.673e+01  -2.509 0.012129 *  
## monNis                     -1.154e+02  2.879e+01  -4.007 6.18e-05 ***
## monOca                      3.712e+01  2.854e+01   1.301 0.193375    
## monŞub                      7.913e+01  3.004e+01   2.634 0.008442 ** 
## monTem                      5.593e+01  2.560e+01   2.185 0.028920 *  
## hour1                       1.039e+02  3.610e+01   2.878 0.004006 ** 
## hour2                       7.501e+01  3.610e+01   2.078 0.037714 *  
## hour3                       5.233e+01  3.611e+01   1.449 0.147333    
## hour4                       2.630e+01  3.613e+01   0.728 0.466547    
## hour5                       2.223e+01  3.616e+01   0.615 0.538612    
## hour6                      -3.988e+01  3.617e+01  -1.102 0.270266    
## hour7                      -3.044e+01  3.615e+01  -0.842 0.399718    
## hour8                       6.724e+01  3.628e+01   1.854 0.063806 .  
## hour9                       1.829e+02  3.640e+01   5.025 5.08e-07 ***
## hour10                      1.235e+02  3.648e+01   3.385 0.000712 ***
## hour11                      1.442e+02  3.667e+01   3.933 8.41e-05 ***
## hour12                     -1.827e+01  3.676e+01  -0.497 0.619150    
## hour13                     -8.566e+01  3.686e+01  -2.324 0.020148 *  
## hour14                      4.608e+01  3.700e+01   1.245 0.212964    
## hour15                      2.940e+01  3.704e+01   0.794 0.427371    
## hour16                      4.964e+00  3.697e+01   0.134 0.893200    
## hour17                      5.354e+01  3.701e+01   1.447 0.147979    
## hour18                      8.280e+01  3.671e+01   2.256 0.024094 *  
## hour19                      8.640e+01  3.653e+01   2.365 0.018037 *  
## hour20                      1.185e+02  3.637e+01   3.259 0.001120 ** 
## hour21                      1.625e+02  3.629e+01   4.479 7.53e-06 ***
## hour22                      1.377e+02  3.617e+01   3.807 0.000141 ***
## hour23                      1.175e+02  3.611e+01   3.255 0.001136 ** 
## prevday                     3.316e-01  5.598e-03  59.232  < 2e-16 ***
## prevweek                    6.486e-02  5.293e-03  12.253  < 2e-16 ***
## loc3tmp2ag                  2.664e+02  1.849e+01  14.410  < 2e-16 ***
## loc4wdir:loc7wdir           2.978e+02  1.699e+01  17.524  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.727e+03  1.598e+02  23.326  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.626e+03  7.360e+01  22.094  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   9.385e+01  2.044e+01   4.591 4.44e-06 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.424e+03  9.433e+01  15.092  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.396e+02  1.915e+01  22.952  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 852.5 on 26757 degrees of freedom
## Multiple R-squared:  0.343,  Adjusted R-squared:  0.3418 
## F-statistic: 279.4 on 50 and 26757 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_7,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_TMP_2.m.above.ground+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_TCDC_low.cloud.layer+loc4_ws_10m+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_RH_2.m.above.ground+loc6_TCDC_low.cloud.layer+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc4ws:=as.numeric(loc4_ws_10m<3.5)]
tmp2[,loc6rh:=as.numeric(loc6_RH_2.m.above.ground<40)]
tmp2[,loc1tmp2ag:=as.numeric(loc1_TMP_2.m.above.ground>=15)]

lm2_8=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh+loc4ws:loc6rh:loc1tmp2ag,
                 tmp2)
summary(lm2_8)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh + loc4ws:loc6rh:loc1tmp2ag, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4935.7  -506.7    -2.3   490.3  4654.7 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.579e+02  3.628e+01  -9.866  < 2e-16 ***
## trnd                        1.422e-02  7.244e-04  19.624  < 2e-16 ***
## w_dayCum                    5.407e+01  1.945e+01   2.779 0.005449 ** 
## w_dayÇar                    9.054e+01  1.946e+01   4.653 3.28e-06 ***
## w_dayPaz                   -1.543e+02  1.950e+01  -7.912 2.62e-15 ***
## w_dayPer                    1.376e+02  1.947e+01   7.070 1.58e-12 ***
## w_dayPts                    2.133e+02  1.957e+01  10.898  < 2e-16 ***
## w_daySal                    1.526e+02  1.947e+01   7.837 4.79e-15 ***
## monAra                      8.991e+01  2.894e+01   3.106 0.001897 ** 
## monEki                      7.890e+01  2.726e+01   2.894 0.003805 ** 
## monEyl                     -8.390e+01  2.632e+01  -3.188 0.001434 ** 
## monHaz                      1.208e+02  2.638e+01   4.579 4.69e-06 ***
## monKas                      1.124e+02  2.866e+01   3.920 8.86e-05 ***
## monMar                      2.759e+01  2.916e+01   0.946 0.344030    
## monMay                     -4.724e+01  2.669e+01  -1.770 0.076703 .  
## monNis                     -8.934e+01  2.877e+01  -3.105 0.001903 ** 
## monOca                      6.580e+01  2.853e+01   2.306 0.021111 *  
## monŞub                      1.077e+02  3.003e+01   3.586 0.000336 ***
## monTem                      8.342e+01  2.561e+01   3.257 0.001125 ** 
## hour1                       1.047e+02  3.598e+01   2.910 0.003615 ** 
## hour2                       7.740e+01  3.598e+01   2.151 0.031495 *  
## hour3                       5.511e+01  3.600e+01   1.531 0.125829    
## hour4                       3.068e+01  3.601e+01   0.852 0.394279    
## hour5                       2.499e+01  3.604e+01   0.693 0.488073    
## hour6                      -3.602e+01  3.606e+01  -0.999 0.317875    
## hour7                      -2.749e+01  3.603e+01  -0.763 0.445465    
## hour8                       6.110e+01  3.616e+01   1.689 0.091146 .  
## hour9                       1.673e+02  3.630e+01   4.609 4.06e-06 ***
## hour10                      1.008e+02  3.641e+01   2.768 0.005636 ** 
## hour11                      1.153e+02  3.662e+01   3.150 0.001635 ** 
## hour12                     -4.214e+01  3.669e+01  -1.148 0.250791    
## hour13                     -9.858e+01  3.676e+01  -2.682 0.007327 ** 
## hour14                      4.826e+01  3.688e+01   1.308 0.190768    
## hour15                      4.126e+01  3.694e+01   1.117 0.263973    
## hour16                      1.883e+01  3.687e+01   0.511 0.609622    
## hour17                      7.158e+01  3.692e+01   1.939 0.052513 .  
## hour18                      1.005e+02  3.662e+01   2.744 0.006070 ** 
## hour19                      9.730e+01  3.643e+01   2.671 0.007568 ** 
## hour20                      1.208e+02  3.626e+01   3.331 0.000867 ***
## hour21                      1.590e+02  3.618e+01   4.395 1.11e-05 ***
## hour22                      1.340e+02  3.605e+01   3.717 0.000202 ***
## hour23                      1.144e+02  3.600e+01   3.178 0.001483 ** 
## prevday                     3.290e-01  5.584e-03  58.911  < 2e-16 ***
## prevweek                    6.705e-02  5.279e-03  12.701  < 2e-16 ***
## loc3tmp2ag                  2.267e+02  1.868e+01  12.138  < 2e-16 ***
## loc4wdir:loc7wdir           2.680e+02  1.709e+01  15.683  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.757e+03  1.593e+02  23.585  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.658e+03  7.341e+01  22.580  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   8.935e+01  2.038e+01   4.383 1.17e-05 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.366e+03  9.413e+01  14.510  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.415e+02  1.909e+01  23.124  < 2e-16 ***
## loc4ws:loc6rh:loc1tmp2ag    3.255e+02  2.487e+01  13.084  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 849.8 on 26756 degrees of freedom
## Multiple R-squared:  0.3472, Adjusted R-squared:  0.3459 
## F-statistic:   279 on 51 and 26756 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_8,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_TMP_2.m.above.ground+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_TCDC_low.cloud.layer+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+loc5_ws_10m+
         loc6_DSWRF_surface+loc6_TCDC_low.cloud.layer+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_RH_2.m.above.ground+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc5ws:=as.numeric(loc5_ws_10m<2.4)]
tmp2[,loc2tmp2ag:=as.numeric(loc2_TMP_2.m.above.ground<2.3)]
tmp2[,loc7rh2:=as.numeric(loc7_RH_2.m.above.ground<63)]

lm2_9=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh+loc4ws:loc6rh:loc1tmp2ag+loc5ws:loc2tmp2ag:loc7rh2,
                 tmp2)
summary(lm2_9)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh + loc4ws:loc6rh:loc1tmp2ag + loc5ws:loc2tmp2ag:loc7rh2, 
##     data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4938.5  -506.4    -2.0   491.1  4657.6 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.558e+02  3.626e+01  -9.813  < 2e-16 ***
## trnd                        1.401e-02  7.250e-04  19.318  < 2e-16 ***
## w_dayCum                    5.363e+01  1.944e+01   2.758 0.005813 ** 
## w_dayÇar                    9.175e+01  1.945e+01   4.717 2.40e-06 ***
## w_dayPaz                   -1.529e+02  1.949e+01  -7.842 4.62e-15 ***
## w_dayPer                    1.377e+02  1.946e+01   7.075 1.53e-12 ***
## w_dayPts                    2.131e+02  1.956e+01  10.895  < 2e-16 ***
## w_daySal                    1.541e+02  1.946e+01   7.920 2.47e-15 ***
## monAra                      8.825e+01  2.893e+01   3.050 0.002287 ** 
## monEki                      7.901e+01  2.725e+01   2.900 0.003739 ** 
## monEyl                     -8.387e+01  2.630e+01  -3.189 0.001431 ** 
## monHaz                      1.203e+02  2.637e+01   4.564 5.03e-06 ***
## monKas                      1.130e+02  2.864e+01   3.944 8.05e-05 ***
## monMar                      2.807e+01  2.914e+01   0.963 0.335428    
## monMay                     -4.805e+01  2.667e+01  -1.801 0.071646 .  
## monNis                     -8.982e+01  2.875e+01  -3.124 0.001789 ** 
## monOca                      5.901e+01  2.854e+01   2.067 0.038718 *  
## monŞub                      1.073e+02  3.001e+01   3.577 0.000349 ***
## monTem                      8.331e+01  2.559e+01   3.255 0.001135 ** 
## hour1                       1.042e+02  3.596e+01   2.898 0.003758 ** 
## hour2                       7.803e+01  3.597e+01   2.170 0.030044 *  
## hour3                       5.586e+01  3.598e+01   1.553 0.120526    
## hour4                       3.092e+01  3.599e+01   0.859 0.390276    
## hour5                       2.535e+01  3.603e+01   0.704 0.481652    
## hour6                      -3.395e+01  3.604e+01  -0.942 0.346298    
## hour7                      -2.587e+01  3.601e+01  -0.718 0.472477    
## hour8                       5.946e+01  3.615e+01   1.645 0.099999 .  
## hour9                       1.668e+02  3.628e+01   4.598 4.29e-06 ***
## hour10                      1.014e+02  3.639e+01   2.786 0.005336 ** 
## hour11                      1.165e+02  3.660e+01   3.182 0.001465 ** 
## hour12                     -4.108e+01  3.667e+01  -1.120 0.262666    
## hour13                     -9.753e+01  3.674e+01  -2.655 0.007944 ** 
## hour14                      4.932e+01  3.686e+01   1.338 0.180981    
## hour15                      4.230e+01  3.692e+01   1.146 0.251818    
## hour16                      1.989e+01  3.685e+01   0.540 0.589384    
## hour17                      7.265e+01  3.690e+01   1.969 0.048953 *  
## hour18                      1.017e+02  3.660e+01   2.778 0.005466 ** 
## hour19                      9.883e+01  3.641e+01   2.714 0.006643 ** 
## hour20                      1.220e+02  3.624e+01   3.366 0.000764 ***
## hour21                      1.598e+02  3.616e+01   4.419 9.96e-06 ***
## hour22                      1.350e+02  3.604e+01   3.745 0.000181 ***
## hour23                      1.149e+02  3.598e+01   3.194 0.001403 ** 
## prevday                     3.291e-01  5.581e-03  58.958  < 2e-16 ***
## prevweek                    6.689e-02  5.276e-03  12.677  < 2e-16 ***
## loc3tmp2ag                  2.263e+02  1.867e+01  12.121  < 2e-16 ***
## loc4wdir:loc7wdir           2.678e+02  1.708e+01  15.680  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.758e+03  1.592e+02  23.605  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.658e+03  7.337e+01  22.600  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   7.994e+01  2.044e+01   3.910 9.25e-05 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.367e+03  9.408e+01  14.526  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.429e+02  1.908e+01  23.208  < 2e-16 ***
## loc4ws:loc6rh:loc1tmp2ag    3.256e+02  2.486e+01  13.096  < 2e-16 ***
## loc5ws:loc2tmp2ag:loc7rh2   6.218e+02  1.139e+02   5.460 4.81e-08 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 849.3 on 26755 degrees of freedom
## Multiple R-squared:  0.3479, Adjusted R-squared:  0.3466 
## F-statistic: 274.5 on 52 and 26755 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_9,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+loc1_ws_10m+
         loc2_DSWRF_surface+loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_TCDC_low.cloud.layer+
         loc5_DSWRF_surface+loc5_RH_2.m.above.ground+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+
         loc6_DSWRF_surface+loc6_TCDC_low.cloud.layer+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc5rh:=as.numeric(loc5_RH_2.m.above.ground<16)]
tmp2[,loc1ws:=as.numeric(loc1_ws_10m>=3.9)]
tmp2[,loc2dswrf:=as.numeric(loc2_DSWRF_surface<841)]

lm2_10=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh+loc4ws:loc6rh:loc1tmp2ag+loc5ws:loc2tmp2ag:loc7rh2+loc5rh:loc1ws:loc2dswrf,
                 tmp2)
summary(lm2_10)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh + loc4ws:loc6rh:loc1tmp2ag + loc5ws:loc2tmp2ag:loc7rh2 + 
##     loc5rh:loc1ws:loc2dswrf, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4930.8  -504.4    -2.4   490.0  4661.5 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.683e+02  3.621e+01 -10.171  < 2e-16 ***
## trnd                        1.406e-02  7.236e-04  19.435  < 2e-16 ***
## w_dayCum                    5.274e+01  1.940e+01   2.718 0.006569 ** 
## w_dayÇar                    9.070e+01  1.941e+01   4.673 2.98e-06 ***
## w_dayPaz                   -1.538e+02  1.946e+01  -7.904 2.80e-15 ***
## w_dayPer                    1.353e+02  1.942e+01   6.968 3.28e-12 ***
## w_dayPts                    2.107e+02  1.952e+01  10.793  < 2e-16 ***
## w_daySal                    1.512e+02  1.942e+01   7.784 7.28e-15 ***
## monAra                      1.032e+02  2.891e+01   3.571 0.000357 ***
## monEki                      9.234e+01  2.722e+01   3.392 0.000695 ***
## monEyl                     -7.390e+01  2.627e+01  -2.813 0.004904 ** 
## monHaz                      1.380e+02  2.637e+01   5.233 1.68e-07 ***
## monKas                      1.274e+02  2.862e+01   4.452 8.53e-06 ***
## monMar                      4.284e+01  2.912e+01   1.471 0.141219    
## monMay                     -3.354e+01  2.666e+01  -1.258 0.208302    
## monNis                     -7.557e+01  2.873e+01  -2.631 0.008529 ** 
## monOca                      7.411e+01  2.852e+01   2.598 0.009370 ** 
## monŞub                      1.225e+02  2.998e+01   4.084 4.44e-05 ***
## monTem                      1.041e+02  2.562e+01   4.062 4.87e-05 ***
## hour1                       1.044e+02  3.589e+01   2.909 0.003628 ** 
## hour2                       7.807e+01  3.589e+01   2.175 0.029641 *  
## hour3                       5.581e+01  3.591e+01   1.554 0.120106    
## hour4                       3.081e+01  3.592e+01   0.858 0.391020    
## hour5                       2.516e+01  3.595e+01   0.700 0.483975    
## hour6                      -3.428e+01  3.597e+01  -0.953 0.340672    
## hour7                      -2.598e+01  3.594e+01  -0.723 0.469811    
## hour8                       5.921e+01  3.607e+01   1.641 0.100709    
## hour9                       1.669e+02  3.621e+01   4.609 4.06e-06 ***
## hour10                      1.014e+02  3.632e+01   2.792 0.005241 ** 
## hour11                      1.171e+02  3.653e+01   3.205 0.001350 ** 
## hour12                     -4.266e+01  3.660e+01  -1.166 0.243727    
## hour13                     -1.023e+02  3.667e+01  -2.790 0.005267 ** 
## hour14                      4.503e+01  3.679e+01   1.224 0.220961    
## hour15                      3.558e+01  3.685e+01   0.966 0.334295    
## hour16                      7.525e+00  3.680e+01   0.204 0.837981    
## hour17                      6.698e+01  3.683e+01   1.819 0.068957 .  
## hour18                      1.011e+02  3.652e+01   2.768 0.005642 ** 
## hour19                      1.015e+02  3.634e+01   2.794 0.005215 ** 
## hour20                      1.231e+02  3.617e+01   3.402 0.000669 ***
## hour21                      1.606e+02  3.608e+01   4.452 8.54e-06 ***
## hour22                      1.353e+02  3.596e+01   3.763 0.000168 ***
## hour23                      1.152e+02  3.591e+01   3.209 0.001332 ** 
## prevday                     3.283e-01  5.571e-03  58.930  < 2e-16 ***
## prevweek                    6.614e-02  5.266e-03  12.559  < 2e-16 ***
## loc3tmp2ag                  2.211e+02  1.864e+01  11.862  < 2e-16 ***
## loc4wdir:loc7wdir           2.687e+02  1.705e+01  15.765  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.575e+03  1.598e+02  22.364  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.508e+03  7.463e+01  20.199  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   7.798e+01  2.040e+01   3.822 0.000133 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.391e+03  9.392e+01  14.815  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.449e+02  1.905e+01  23.359  < 2e-16 ***
## loc4ws:loc6rh:loc1tmp2ag    3.345e+02  2.483e+01  13.473  < 2e-16 ***
## loc5ws:loc2tmp2ag:loc7rh2   6.214e+02  1.137e+02   5.467 4.61e-08 ***
## loc5rh:loc1ws:loc2dswrf     1.160e+03  1.112e+02  10.435  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 847.6 on 26754 degrees of freedom
## Multiple R-squared:  0.3505, Adjusted R-squared:  0.3492 
## F-statistic: 272.4 on 53 and 26754 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_10,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~
         loc1_DSWRF_surface+loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+
         loc2_RH_2.m.above.ground+loc2_TCDC_low.cloud.layer+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_TCDC_low.cloud.layer+
         loc5_DSWRF_surface+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+
         loc6_DSWRF_surface+loc6_TCDC_low.cloud.layer+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc6tcdc:=as.numeric(loc6_TCDC_low.cloud.layer>=84)]
tmp2[,loc1dswrf:=as.numeric(loc1_DSWRF_surface>=90)]
tmp2[,loc2rh2:=as.numeric(loc2_RH_2.m.above.ground>=91)]

lm2_11=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh+loc4ws:loc6rh:loc1tmp2ag+loc5ws:loc2tmp2ag:loc7rh2+loc5rh:loc1ws:loc2dswrf
         +loc6tcdc:loc1dswrf:loc2rh2,
                 tmp2)
summary(lm2_11)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh + loc4ws:loc6rh:loc1tmp2ag + loc5ws:loc2tmp2ag:loc7rh2 + 
##     loc5rh:loc1ws:loc2dswrf + loc6tcdc:loc1dswrf:loc2rh2, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4927.5  -505.1    -1.9   490.9  4594.2 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.671e+02  3.614e+01 -10.158  < 2e-16 ***
## trnd                        1.404e-02  7.221e-04  19.437  < 2e-16 ***
## w_dayCum                    5.393e+01  1.937e+01   2.785 0.005359 ** 
## w_dayÇar                    9.022e+01  1.937e+01   4.658 3.21e-06 ***
## w_dayPaz                   -1.543e+02  1.942e+01  -7.949 1.96e-15 ***
## w_dayPer                    1.361e+02  1.938e+01   7.024 2.20e-12 ***
## w_dayPts                    2.067e+02  1.949e+01  10.607  < 2e-16 ***
## w_daySal                    1.508e+02  1.938e+01   7.781 7.43e-15 ***
## monAra                      9.568e+01  2.886e+01   3.315 0.000916 ***
## monEki                      9.305e+01  2.717e+01   3.425 0.000616 ***
## monEyl                     -7.362e+01  2.621e+01  -2.809 0.004979 ** 
## monHaz                      1.381e+02  2.631e+01   5.249 1.54e-07 ***
## monKas                      1.297e+02  2.856e+01   4.542 5.60e-06 ***
## monMar                      4.380e+01  2.906e+01   1.507 0.131723    
## monMay                     -3.203e+01  2.660e+01  -1.204 0.228620    
## monNis                     -7.300e+01  2.867e+01  -2.546 0.010904 *  
## monOca                      7.250e+01  2.846e+01   2.547 0.010874 *  
## monŞub                      1.193e+02  2.993e+01   3.985 6.76e-05 ***
## monTem                      1.039e+02  2.557e+01   4.064 4.83e-05 ***
## hour1                       1.046e+02  3.582e+01   2.921 0.003492 ** 
## hour2                       7.824e+01  3.582e+01   2.184 0.028950 *  
## hour3                       5.590e+01  3.583e+01   1.560 0.118808    
## hour4                       3.076e+01  3.585e+01   0.858 0.390833    
## hour5                       2.513e+01  3.588e+01   0.700 0.483687    
## hour6                      -3.438e+01  3.590e+01  -0.958 0.338255    
## hour7                      -2.612e+01  3.587e+01  -0.728 0.466526    
## hour8                       5.942e+01  3.600e+01   1.651 0.098851 .  
## hour9                       1.675e+02  3.613e+01   4.635 3.59e-06 ***
## hour10                      9.775e+01  3.624e+01   2.697 0.007001 ** 
## hour11                      1.153e+02  3.645e+01   3.164 0.001558 ** 
## hour12                     -4.435e+01  3.652e+01  -1.214 0.224615    
## hour13                     -1.054e+02  3.660e+01  -2.881 0.003966 ** 
## hour14                      3.938e+01  3.672e+01   1.072 0.283549    
## hour15                      3.005e+01  3.678e+01   0.817 0.413907    
## hour16                      4.650e+00  3.673e+01   0.127 0.899238    
## hour17                      6.705e+01  3.675e+01   1.824 0.068101 .  
## hour18                      9.807e+01  3.645e+01   2.690 0.007144 ** 
## hour19                      1.011e+02  3.626e+01   2.788 0.005304 ** 
## hour20                      1.211e+02  3.609e+01   3.355 0.000796 ***
## hour21                      1.600e+02  3.601e+01   4.444 8.86e-06 ***
## hour22                      1.356e+02  3.589e+01   3.778 0.000158 ***
## hour23                      1.154e+02  3.583e+01   3.222 0.001277 ** 
## prevday                     3.278e-01  5.560e-03  58.960  < 2e-16 ***
## prevweek                    6.562e-02  5.256e-03  12.485  < 2e-16 ***
## loc3tmp2ag                  2.230e+02  1.860e+01  11.988  < 2e-16 ***
## loc4wdir:loc7wdir           2.696e+02  1.701e+01  15.850  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.578e+03  1.595e+02  22.428  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.509e+03  7.448e+01  20.262  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   7.891e+01  2.036e+01   3.875 0.000107 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.393e+03  9.373e+01  14.866  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.350e+02  1.903e+01  22.855  < 2e-16 ***
## loc4ws:loc6rh:loc1tmp2ag    3.342e+02  2.478e+01  13.487  < 2e-16 ***
## loc5ws:loc2tmp2ag:loc7rh2   6.243e+02  1.134e+02   5.504 3.74e-08 ***
## loc5rh:loc1ws:loc2dswrf     1.162e+03  1.110e+02  10.468  < 2e-16 ***
## loc6tcdc:loc1dswrf:loc2rh2  1.595e+03  1.527e+02  10.444  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 845.9 on 26753 degrees of freedom
## Multiple R-squared:  0.3532, Adjusted R-squared:  0.3519 
## F-statistic: 270.5 on 54 and 26753 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_11,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~
         loc1_RH_2.m.above.ground+loc1_TCDC_low.cloud.layer+
         loc2_TCDC_low.cloud.layer+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+loc3_ws_10m+
         loc4_DSWRF_surface+loc4_TCDC_low.cloud.layer+
         loc5_DSWRF_surface+loc5_TCDC_low.cloud.layer+loc5_wdir_10m+
         loc6_DSWRF_surface+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

tmp2[,loc3ws:=as.numeric(loc3_ws_10m<1.1)]
tmp2[,loc1tcdc:=as.numeric(loc1_TCDC_low.cloud.layer>=63)]
tmp2[,loc5wdir:=as.numeric(loc5_wdir_10m<69)]

lm2_12=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh+loc4ws:loc6rh:loc1tmp2ag+loc5ws:loc2tmp2ag:loc7rh2+loc5rh:loc1ws:loc2dswrf
         +loc6tcdc:loc1dswrf:loc2rh2+loc3ws:loc1tcdc:loc5wdir,
                 tmp2)
summary(lm2_12)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh + loc4ws:loc6rh:loc1tmp2ag + loc5ws:loc2tmp2ag:loc7rh2 + 
##     loc5rh:loc1ws:loc2dswrf + loc6tcdc:loc1dswrf:loc2rh2 + loc3ws:loc1tcdc:loc5wdir, 
##     data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4931.9  -504.3    -1.2   492.1  4599.4 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.658e+02  3.609e+01 -10.136  < 2e-16 ***
## trnd                        1.389e-02  7.214e-04  19.260  < 2e-16 ***
## w_dayCum                    5.387e+01  1.934e+01   2.786 0.005345 ** 
## w_dayÇar                    9.097e+01  1.934e+01   4.702 2.58e-06 ***
## w_dayPaz                   -1.536e+02  1.939e+01  -7.919 2.48e-15 ***
## w_dayPer                    1.343e+02  1.936e+01   6.937 4.10e-12 ***
## w_dayPts                    2.054e+02  1.946e+01  10.555  < 2e-16 ***
## w_daySal                    1.518e+02  1.936e+01   7.842 4.60e-15 ***
## monAra                      9.180e+01  2.882e+01   3.185 0.001450 ** 
## monEki                      9.345e+01  2.713e+01   3.444 0.000574 ***
## monEyl                     -7.342e+01  2.618e+01  -2.805 0.005042 ** 
## monHaz                      1.381e+02  2.628e+01   5.253 1.51e-07 ***
## monKas                      1.286e+02  2.852e+01   4.510 6.52e-06 ***
## monMar                      4.000e+01  2.902e+01   1.378 0.168135    
## monMay                     -3.251e+01  2.657e+01  -1.224 0.221043    
## monNis                     -7.400e+01  2.863e+01  -2.584 0.009764 ** 
## monOca                      6.647e+01  2.844e+01   2.337 0.019426 *  
## monŞub                      1.180e+02  2.989e+01   3.949 7.86e-05 ***
## monTem                      1.039e+02  2.553e+01   4.068 4.75e-05 ***
## hour1                       1.056e+02  3.577e+01   2.953 0.003151 ** 
## hour2                       7.717e+01  3.577e+01   2.157 0.030989 *  
## hour3                       5.794e+01  3.579e+01   1.619 0.105458    
## hour4                       3.175e+01  3.580e+01   0.887 0.375113    
## hour5                       2.610e+01  3.583e+01   0.728 0.466468    
## hour6                      -3.233e+01  3.585e+01  -0.902 0.367117    
## hour7                      -2.515e+01  3.582e+01  -0.702 0.482564    
## hour8                       5.849e+01  3.595e+01   1.627 0.103795    
## hour9                       1.665e+02  3.609e+01   4.614 3.96e-06 ***
## hour10                      9.722e+01  3.620e+01   2.686 0.007237 ** 
## hour11                      1.157e+02  3.641e+01   3.179 0.001479 ** 
## hour12                     -4.388e+01  3.648e+01  -1.203 0.229001    
## hour13                     -1.037e+02  3.655e+01  -2.839 0.004534 ** 
## hour14                      4.131e+01  3.667e+01   1.126 0.260039    
## hour15                      3.201e+01  3.673e+01   0.872 0.383481    
## hour16                      5.314e+00  3.668e+01   0.145 0.884810    
## hour17                      6.746e+01  3.670e+01   1.838 0.066094 .  
## hour18                      9.763e+01  3.640e+01   2.682 0.007326 ** 
## hour19                      1.004e+02  3.622e+01   2.772 0.005579 ** 
## hour20                      1.204e+02  3.605e+01   3.341 0.000834 ***
## hour21                      1.613e+02  3.596e+01   4.485 7.32e-06 ***
## hour22                      1.377e+02  3.584e+01   3.842 0.000122 ***
## hour23                      1.175e+02  3.579e+01   3.284 0.001026 ** 
## prevday                     3.276e-01  5.552e-03  59.000  < 2e-16 ***
## prevweek                    6.601e-02  5.249e-03  12.575  < 2e-16 ***
## loc3tmp2ag                  2.231e+02  1.858e+01  12.012  < 2e-16 ***
## loc4wdir:loc7wdir           2.689e+02  1.699e+01  15.827  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.579e+03  1.593e+02  22.464  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.510e+03  7.438e+01  20.305  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   7.994e+01  2.034e+01   3.931 8.49e-05 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.394e+03  9.361e+01  14.889  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.340e+02  1.901e+01  22.835  < 2e-16 ***
## loc4ws:loc6rh:loc1tmp2ag    3.346e+02  2.474e+01  13.522  < 2e-16 ***
## loc5ws:loc2tmp2ag:loc7rh2   6.304e+02  1.133e+02   5.564 2.65e-08 ***
## loc5rh:loc1ws:loc2dswrf     1.161e+03  1.108e+02  10.478  < 2e-16 ***
## loc6tcdc:loc1dswrf:loc2rh2  1.448e+03  1.534e+02   9.441  < 2e-16 ***
## loc3ws:loc1tcdc:loc5wdir    1.165e+03  1.365e+02   8.534  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 844.8 on 26752 degrees of freedom
## Multiple R-squared:  0.3549, Adjusted R-squared:  0.3536 
## F-statistic: 267.6 on 55 and 26752 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_12,tmp2)]
#head(tmp)
ggplot(tmp2[Date=='2019-05-08'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2[,residual:=actual-predicted]


fit_res_tree=rpart(residual~
         loc1_RH_2.m.above.ground+
         loc2_TCDC_low.cloud.layer+loc2_wdir_10m+loc2_ws_10m+
         loc3_DSWRF_surface+loc3_RH_2.m.above.ground+
         loc4_DSWRF_surface+loc4_TCDC_low.cloud.layer+
         loc5_DSWRF_surface+loc5_TCDC_low.cloud.layer+
         loc6_DSWRF_surface+loc6_wdir_10m+loc6_ws_10m+
         loc7_DSWRF_surface+loc7_TCDC_low.cloud.layer,tmp2,
                   control=rpart.control(cp=0,maxdepth=3))

fancyRpartPlot(fit_res_tree)

After comparision between models, deciding to continue with the 13th model
Submit results depends on this model

tmp2[,loc3rh:=as.numeric(loc3_RH_2.m.above.ground<25)]
tmp2[,loc5tcdc:=as.numeric(loc5_TCDC_low.cloud.layer<0.15)]
tmp2[,loc1rh:=as.numeric(loc1_RH_2.m.above.ground>=15)]

lm2_13=lm(net~trnd+w_day+mon+hour+prevday+prevweek+loc7tmp2ag:pv:loc7tmp2ag2+loc5tmp2ag:pd:workday+loc4tmp2ag:whour:loc3wdir+loc6tmp2ag:loc7ws:loc1wdir
         +loc4wdir:loc7wdir+loc3tmp2ag+loc3tcdc:mon1:loc4rh+loc4ws:loc6rh:loc1tmp2ag+loc5ws:loc2tmp2ag:loc7rh2+loc5rh:loc1ws:loc2dswrf
         +loc6tcdc:loc1dswrf:loc2rh2+loc3ws:loc1tcdc:loc5wdir+loc3rh:loc5tcdc:loc1rh,
                 tmp2)
summary(lm2_13)
## 
## Call:
## lm(formula = net ~ trnd + w_day + mon + hour + prevday + prevweek + 
##     loc7tmp2ag:pv:loc7tmp2ag2 + loc5tmp2ag:pd:workday + loc4tmp2ag:whour:loc3wdir + 
##     loc6tmp2ag:loc7ws:loc1wdir + loc4wdir:loc7wdir + loc3tmp2ag + 
##     loc3tcdc:mon1:loc4rh + loc4ws:loc6rh:loc1tmp2ag + loc5ws:loc2tmp2ag:loc7rh2 + 
##     loc5rh:loc1ws:loc2dswrf + loc6tcdc:loc1dswrf:loc2rh2 + loc3ws:loc1tcdc:loc5wdir + 
##     loc3rh:loc5tcdc:loc1rh, data = tmp2)
## 
## Residuals:
##     Min      1Q  Median      3Q     Max 
## -4882.9  -505.1    -1.5   493.3  4688.8 
## 
## Coefficients:
##                              Estimate Std. Error t value Pr(>|t|)    
## (Intercept)                -3.727e+02  3.601e+01 -10.349  < 2e-16 ***
## trnd                        1.389e-02  7.196e-04  19.306  < 2e-16 ***
## w_dayCum                    5.607e+01  1.929e+01   2.906 0.003666 ** 
## w_dayÇar                    8.613e+01  1.930e+01   4.462 8.16e-06 ***
## w_dayPaz                   -1.550e+02  1.934e+01  -8.015 1.15e-15 ***
## w_dayPer                    1.335e+02  1.931e+01   6.912 4.88e-12 ***
## w_dayPts                    1.998e+02  1.942e+01  10.290  < 2e-16 ***
## w_daySal                    1.480e+02  1.931e+01   7.662 1.90e-14 ***
## monAra                      1.095e+02  2.880e+01   3.802 0.000144 ***
## monEki                      1.007e+02  2.707e+01   3.720 0.000200 ***
## monEyl                     -5.841e+01  2.615e+01  -2.234 0.025496 *  
## monHaz                      1.678e+02  2.635e+01   6.368 1.95e-10 ***
## monKas                      1.438e+02  2.849e+01   5.049 4.46e-07 ***
## monMar                      5.443e+01  2.898e+01   1.878 0.060394 .  
## monMay                     -3.177e+01  2.650e+01  -1.199 0.230651    
## monNis                     -6.222e+01  2.858e+01  -2.177 0.029509 *  
## monOca                      8.441e+01  2.841e+01   2.971 0.002970 ** 
## monŞub                      1.345e+02  2.985e+01   4.506 6.64e-06 ***
## monTem                      1.189e+02  2.551e+01   4.661 3.17e-06 ***
## hour1                       1.056e+02  3.569e+01   2.960 0.003082 ** 
## hour2                       7.598e+01  3.569e+01   2.129 0.033263 *  
## hour3                       5.697e+01  3.570e+01   1.596 0.110584    
## hour4                       3.044e+01  3.572e+01   0.852 0.394106    
## hour5                       2.401e+01  3.575e+01   0.672 0.501753    
## hour6                      -3.485e+01  3.577e+01  -0.974 0.329837    
## hour7                      -2.644e+01  3.574e+01  -0.740 0.459403    
## hour8                       5.715e+01  3.587e+01   1.593 0.111072    
## hour9                       1.673e+02  3.600e+01   4.647 3.38e-06 ***
## hour10                      9.874e+01  3.611e+01   2.734 0.006255 ** 
## hour11                      1.154e+02  3.632e+01   3.177 0.001489 ** 
## hour12                     -4.996e+01  3.639e+01  -1.373 0.169805    
## hour13                     -1.227e+02  3.650e+01  -3.360 0.000779 ***
## hour14                      1.547e+01  3.666e+01   0.422 0.673070    
## hour15                      2.584e+00  3.673e+01   0.070 0.943925    
## hour16                     -2.736e+01  3.670e+01  -0.746 0.455937    
## hour17                      3.577e+01  3.672e+01   0.974 0.329968    
## hour18                      7.311e+01  3.638e+01   2.009 0.044498 *  
## hour19                      8.868e+01  3.614e+01   2.454 0.014154 *  
## hour20                      1.166e+02  3.596e+01   3.242 0.001190 ** 
## hour21                      1.619e+02  3.588e+01   4.513 6.41e-06 ***
## hour22                      1.387e+02  3.576e+01   3.878 0.000105 ***
## hour23                      1.189e+02  3.570e+01   3.330 0.000869 ***
## prevday                     3.246e-01  5.545e-03  58.548  < 2e-16 ***
## prevweek                    6.674e-02  5.237e-03  12.745  < 2e-16 ***
## loc3tmp2ag                  1.786e+02  1.894e+01   9.426  < 2e-16 ***
## loc4wdir:loc7wdir           2.534e+02  1.700e+01  14.901  < 2e-16 ***
## loc7tmp2ag:pv:loc7tmp2ag2   3.545e+03  1.589e+02  22.304  < 2e-16 ***
## loc5tmp2ag:pd:workday       1.385e+03  7.502e+01  18.467  < 2e-16 ***
## loc4tmp2ag:whour:loc3wdir   7.301e+01  2.030e+01   3.597 0.000322 ***
## loc6tmp2ag:loc7ws:loc1wdir  1.367e+03  9.341e+01  14.638  < 2e-16 ***
## loc3tcdc:mon1:loc4rh        4.477e+02  1.900e+01  23.563  < 2e-16 ***
## loc4ws:loc6rh:loc1tmp2ag    3.097e+02  2.478e+01  12.496  < 2e-16 ***
## loc5ws:loc2tmp2ag:loc7rh2   6.280e+02  1.130e+02   5.557 2.77e-08 ***
## loc5rh:loc1ws:loc2dswrf     1.098e+03  1.107e+02   9.921  < 2e-16 ***
## loc6tcdc:loc1dswrf:loc2rh2  1.453e+03  1.531e+02   9.495  < 2e-16 ***
## loc3ws:loc1tcdc:loc5wdir    1.166e+03  1.362e+02   8.561  < 2e-16 ***
## loc3rh:loc5tcdc:loc1rh      2.835e+02  2.494e+01  11.368  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 842.8 on 26751 degrees of freedom
## Multiple R-squared:  0.358,  Adjusted R-squared:  0.3567 
## F-statistic: 266.4 on 56 and 26751 DF,  p-value: < 2.2e-16
tmp2[,predicted:=predict(lm2_13,tmp2)]
checkresiduals(lm2_13$residuals)
## Warning in modeldf.default(object): Could not find appropriate degrees of
## freedom for this model.

tmp2[,residuals13 := residuals(lm2_13)]
acf(residuals(lm2_13) , lag.max = 160)

ggplot(tmp2[Date=='2022-01-07'] ,aes(x=datetime)) +
        geom_line(aes(y=actual,color='real')) + 
        geom_line(aes(y=predicted,color='predicted'))

tmp2 = tmp2 %>% 
  mutate(predicted_diretion = 
           case_when(predicted < -50 ~ "Negative" ,
                     predicted>=-50 & predicted<=50 ~ "Neutral" ,
                     predicted> 50 ~ "Positive" )
         )
tmp2= as.data.table(tmp2)
tmp2
##                   datetime       net system_direction loc1_DSWRF_surface
##     1: 2019-01-01 00:00:00 -1934.795         Negative               0.00
##     2: 2019-01-01 01:00:00  -750.866         Negative               0.00
##     3: 2019-01-01 02:00:00  -982.767         Negative               0.00
##     4: 2019-01-01 03:00:00  -764.620         Negative               0.00
##     5: 2019-01-01 04:00:00 -1233.289         Negative               0.00
##    ---                                                                  
## 26804: 2022-01-21 19:00:00     0.000                               55.38
## 26805: 2022-01-21 20:00:00     0.000                               44.30
## 26806: 2022-01-21 21:00:00     0.000                               36.92
## 26807: 2022-01-21 22:00:00     0.000                                0.00
## 26808: 2022-01-21 23:00:00     0.000                                0.00
##        loc1_RH_2.m.above.ground loc1_TCDC_low.cloud.layer
##     1:                 84.20000                   0.00000
##     2:                 84.43333                   3.00000
##     3:                 84.66667                   6.00000
##     4:                 84.90000                   9.00000
##     5:                 87.26667                  29.66667
##    ---                                                   
## 26804:                 97.90000                  99.90000
## 26805:                 98.00000                  99.90000
## 26806:                 98.70000                  99.90000
## 26807:                 98.80000                 100.00000
## 26808:                 98.90000                 100.00000
##        loc1_TMP_2.m.above.ground loc1_wdir_10m loc1_ws_10m loc2_DSWRF_surface
##     1:                    -4.081    244.986564    2.302671               0.00
##     2:                    -3.707    245.382871    2.443606               0.00
##     3:                    -3.333    245.779178    2.584541               0.00
##     4:                    -2.959    246.175485    2.725476               0.00
##     5:                    -2.591    240.651199    2.755832               0.00
##    ---                                                                       
## 26804:                    -1.623      2.212407    1.948613              40.46
## 26805:                    -1.285    357.806092    1.767906              32.36
## 26806:                    -0.937    354.741987    2.657864              26.98
## 26807:                    -0.641      7.030764    3.098056               0.00
## 26808:                    -0.540     27.723707    3.520917               0.00
##        loc2_RH_2.m.above.ground loc2_TCDC_low.cloud.layer
##     1:                 59.80000                       0.0
##     2:                 56.93333                       0.0
##     3:                 54.06667                       0.0
##     4:                 51.20000                       0.0
##     5:                 51.56667                       0.0
##    ---                                                   
## 26804:                 66.80000                      99.9
## 26805:                 68.60000                      99.9
## 26806:                 68.50000                      99.9
## 26807:                 73.90000                     100.0
## 26808:                 79.80000                     100.0
##        loc2_TMP_2.m.above.ground loc2_wdir_10m loc2_ws_10m loc3_DSWRF_surface
##     1:                  3.369000      192.5959    3.057220               0.00
##     2:                  3.333000      192.2694    3.202298               0.00
##     3:                  3.297000      191.9429    3.347376               0.00
##     4:                  3.261000      191.6164    3.492454               0.00
##     5:                  3.365667      191.7771    3.608525               0.00
##    ---                                                                       
## 26804:                  4.347000      208.0302    3.265933             163.96
## 26805:                  4.295000      203.6774    3.355885             131.16
## 26806:                  4.533000      206.7301    3.698566             109.30
## 26807:                  4.649000      211.2813    4.276995               0.00
## 26808:                  4.530000      212.4927    3.987479               0.00
##        loc3_RH_2.m.above.ground loc3_TCDC_low.cloud.layer
##     1:                 84.40000                 1.0000000
##     2:                 85.06667                 0.6666667
##     3:                 85.73333                 0.3333333
##     4:                 86.40000                 0.0000000
##     5:                 87.83333                 1.3333333
##    ---                                                   
## 26804:                 72.40000                 0.0000000
## 26805:                 71.90000                 0.0000000
## 26806:                 72.70000                 0.0000000
## 26807:                 74.60000                 5.0000000
## 26808:                 77.70000                 4.8000000
##        loc3_TMP_2.m.above.ground loc3_wdir_10m loc3_ws_10m loc4_DSWRF_surface
##     1:                 -6.211000     351.73126    1.784915               0.00
##     2:                 -5.950333     350.11411    1.928927               0.00
##     3:                 -5.689667     348.49697    2.072939               0.00
##     4:                 -5.429000     346.87983    2.216951               0.00
##     5:                 -5.094333     337.72332    1.997965               0.00
##    ---                                                                       
## 26804:                 -2.723000      43.76432    3.014659              30.56
## 26805:                 -3.055000      45.90085    2.955315              24.46
## 26806:                 -3.107000      42.20943    2.763192              20.36
## 26807:                 -3.091000      39.53944    2.716282               0.00
## 26808:                 -2.910000      51.42887    2.849656               0.00
##        loc4_RH_2.m.above.ground loc4_TCDC_low.cloud.layer
##     1:                 52.20000                   0.00000
##     2:                 53.90000                   9.00000
##     3:                 55.60000                  18.00000
##     4:                 57.30000                  27.00000
##     5:                 59.76667                  51.33333
##    ---                                                   
## 26804:                 88.20000                  98.10000
## 26805:                 89.20000                  98.50000
## 26806:                 89.20000                  98.70000
## 26807:                 89.30000                 100.00000
## 26808:                 90.10000                 100.00000
##        loc4_TMP_2.m.above.ground loc4_wdir_10m loc4_ws_10m loc5_DSWRF_surface
##     1:                  7.309000      248.9115    5.151732               0.00
##     2:                  7.203000      248.8854    5.512422               0.00
##     3:                  7.097000      248.8594    5.873112               0.00
##     4:                  6.991000      248.8333    6.233802               0.00
##     5:                  6.902333      248.9605    6.574972               0.00
##    ---                                                                       
## 26804:                  9.287000      352.6565    4.262119             116.34
## 26805:                  9.265000      352.4482    4.243415              93.08
## 26806:                  9.233000      350.1066    4.036709              77.56
## 26807:                  9.189000      346.2419    3.577402               0.00
## 26808:                  9.140000      339.1743    3.634154               0.00
##        loc5_RH_2.m.above.ground loc5_TCDC_low.cloud.layer
##     1:                 88.30000                 0.0000000
##     2:                 87.66667                 0.0000000
##     3:                 87.03333                 0.0000000
##     4:                 86.40000                 0.0000000
##     5:                 86.66667                 0.3333333
##    ---                                                   
## 26804:                 84.30000                69.8000000
## 26805:                 81.30000                69.5000000
## 26806:                 86.40000                70.2000000
## 26807:                 89.50000               100.0000000
## 26808:                 94.00000                99.1000000
##        loc5_TMP_2.m.above.ground loc5_wdir_10m loc5_ws_10m loc6_DSWRF_surface
##     1:                 -4.191000     297.65206    3.913716               0.00
##     2:                 -4.070333     295.48408    4.243639               0.00
##     3:                 -3.949667     293.31610    4.573562               0.00
##     4:                 -3.829000     291.14811    4.903485               0.00
##     5:                 -3.864333     290.61532    4.996934               0.00
##    ---                                                                       
## 26804:                  1.997000      19.05118    4.366314             114.36
## 26805:                  2.185000      29.69346    5.556206              91.48
## 26806:                  1.463000      47.68443    4.525578              76.24
## 26807:                  1.679000      63.83293    3.843080               0.00
## 26808:                  1.490000      56.89650    3.948929               0.00
##        loc6_RH_2.m.above.ground loc6_TCDC_low.cloud.layer
##     1:                 92.10000                  28.00000
##     2:                 92.50000                  25.66667
##     3:                 92.90000                  23.33333
##     4:                 93.30000                  21.00000
##     5:                 94.03333                  21.00000
##    ---                                                   
## 26804:                 73.20000                  74.30000
## 26805:                 72.20000                  79.40000
## 26806:                 74.20000                  82.60000
## 26807:                 89.20000                 100.00000
## 26808:                 82.50000                 100.00000
##        loc6_TMP_2.m.above.ground loc6_wdir_10m loc6_ws_10m loc7_DSWRF_surface
##     1:                 -7.041000      280.4415    1.745607               0.00
##     2:                 -7.200333      274.3150    1.962468               0.00
##     3:                 -7.359667      268.1885    2.179329               0.00
##     4:                 -7.519000      262.0619    2.396190               0.00
##     5:                 -7.527667      260.3688    2.522906               0.00
##    ---                                                                       
## 26804:                 -1.953000      324.6584    2.117349               7.32
## 26805:                 -1.895000      312.2008    2.332197               5.86
## 26806:                 -1.907000      331.4124    2.160014               4.88
## 26807:                 -2.371000      300.8462    2.388677               0.00
## 26808:                 -1.210000      345.0344    2.563677               0.00
##        loc7_RH_2.m.above.ground loc7_TCDC_low.cloud.layer
##     1:                 65.80000                         0
##     2:                 66.40000                         0
##     3:                 67.00000                         0
##     4:                 67.60000                         0
##     5:                 67.83333                         0
##    ---                                                   
## 26804:                 94.90000                       100
## 26805:                 94.80000                       100
## 26806:                 95.00000                       100
## 26807:                 95.20000                       100
## 26808:                 95.20000                       100
##        loc7_TMP_2.m.above.ground loc7_wdir_10m loc7_ws_10m  trnd w_day mon hour
##     1:                  7.239000      250.5319    5.861836     1   Sal Oca    0
##     2:                  7.153000      250.9196    6.254371     2   Sal Oca    1
##     3:                  7.067000      251.3073    6.646906     3   Sal Oca    2
##     4:                  6.981000      251.6950    7.039441     4   Sal Oca    3
##     5:                  6.962333      252.8526    7.291198     5   Sal Oca    4
##    ---                                                                         
## 26804:                  3.577000      192.1873    6.417472 26804   Cum Oca   19
## 26805:                  2.815000      181.7018    6.656326 26805   Cum Oca   20
## 26806:                  2.743000      185.1566    6.047797 26806   Cum Oca   21
## 26807:                  2.399000      184.1567    6.081236 26807   Cum Oca   22
## 26808:                  2.180000      182.7795    6.641093 26808   Cum Oca   23
##              Date    actual   predicted   prevday  prevweek   residual
##     1: 2019-01-01 -1934.795  -27.562135  287.9239  287.9239 -1900.6192
##     2: 2019-01-01  -750.866   78.071304  287.9239  287.9239  -822.3306
##     3: 2019-01-01  -982.767   48.444963  287.9239  287.9239 -1025.7931
##     4: 2019-01-01  -764.620   29.445657  287.9239  287.9239  -788.4248
##     5: 2019-01-01 -1233.289    2.931106  287.9239  287.9239 -1230.9230
##    ---                                                                
## 26804: 2022-01-21     0.000  863.904080 2043.4290 -425.3450  -868.6487
## 26805: 2022-01-21     0.000  939.097861 2171.4590 -339.5620  -936.3272
## 26806: 2022-01-21     0.000 1246.179931 2981.0760 -356.4620 -1241.2896
## 26807: 2022-01-21     0.000 1097.096550 2627.5980 -522.6550 -1090.9664
## 26808: 2022-01-21     0.000  269.038245    0.0000  147.8350  -254.2863
##        loc7tmp2ag pv loc7tmp2ag2 loc5tmp2ag pd workday loc4tmp2ag whour
##     1:          0  0           0          0  0       1          1     1
##     2:          0  0           0          0  0       1          1     1
##     3:          0  0           0          0  0       1          1     1
##     4:          0  0           0          0  0       1          1     1
##     5:          0  0           0          0  0       1          1     1
##    ---                                                                 
## 26804:          0  0           0          0  0       1          0     1
## 26805:          0  0           0          0  0       1          0     1
## 26806:          0  0           0          0  1       1          0     1
## 26807:          0  0           0          0  1       1          0     1
## 26808:          0  0           0          0  0       1          0     1
##        loc3wdir loc6tmp2ag loc7ws loc1wdir loc4wdir loc7wdir loc3tmp2ag
##     1:        0          0      0        1        0        1          0
##     2:        0          0      0        1        0        1          0
##     3:        0          0      0        1        0        1          0
##     4:        0          0      0        1        0        1          0
##     5:        0          0      0        1        0        1          0
##    ---                                                                 
## 26804:        1          0      0        0        0        0          0
## 26805:        1          0      0        1        0        0          0
## 26806:        1          0      0        1        0        0          0
## 26807:        1          0      0        0        0        0          0
## 26808:        1          0      0        0        0        0          0
##        loc3tcdc mon1 loc4rh loc4ws loc6rh loc1tmp2ag loc5ws loc2tmp2ag loc7rh2
##     1:        0    1      1      0      0          0      0          0       0
##     2:        0    1      1      0      0          0      0          0       0
##     3:        0    1      1      0      0          0      0          0       0
##     4:        0    1      1      0      0          0      0          0       0
##     5:        0    1      1      0      0          0      0          0       0
##    ---                                                                        
## 26804:        0    1      0      0      0          0      0          0       0
## 26805:        0    1      0      0      0          0      0          0       0
## 26806:        0    1      0      0      0          0      0          0       0
## 26807:        0    1      0      0      0          0      0          0       0
## 26808:        0    1      0      0      0          0      0          0       0
##        loc5rh loc1ws loc2dswrf loc6tcdc loc1dswrf loc2rh2 loc3ws loc1tcdc
##     1:      0      0         1        0         0       0      0        0
##     2:      0      0         1        0         0       0      0        0
##     3:      0      0         1        0         0       0      0        0
##     4:      0      0         1        0         0       0      0        0
##     5:      0      0         1        0         0       0      0        0
##    ---                                                                   
## 26804:      0      0         1        0         0       0      0        1
## 26805:      0      0         1        0         0       0      0        1
## 26806:      0      0         1        0         0       0      0        1
## 26807:      0      0         1        1         0       0      0        1
## 26808:      0      0         1        1         0       0      0        1
##        loc5wdir loc3rh loc5tcdc loc1rh residuals13 predicted_diretion
##     1:        0      0        1      1  -1907.2329            Neutral
##     2:        0      0        1      1   -828.9373           Positive
##     3:        0      0        1      1  -1031.2120            Neutral
##     4:        0      0        1      1   -794.0657            Neutral
##     5:        0      0        0      1  -1236.2201            Neutral
##    ---                                                               
## 26804:        1      0        0      1   -863.9041           Positive
## 26805:        1      0        0      1   -939.0979           Positive
## 26806:        1      0        0      1  -1246.1799           Positive
## 26807:        1      0        0      1  -1097.0965           Positive
## 26808:        1      0        0      1   -269.0382           Positive
submission = tmp2[,list(Date,hour,system_direction,predicted_diretion)]
tail(submission,12)
##           Date hour system_direction predicted_diretion
##  1: 2022-01-21   12                            Positive
##  2: 2022-01-21   13                            Positive
##  3: 2022-01-21   14                            Positive
##  4: 2022-01-21   15                            Positive
##  5: 2022-01-21   16                            Positive
##  6: 2022-01-21   17                            Positive
##  7: 2022-01-21   18                            Positive
##  8: 2022-01-21   19                            Positive
##  9: 2022-01-21   20                            Positive
## 10: 2022-01-21   21                            Positive
## 11: 2022-01-21   22                            Positive
## 12: 2022-01-21   23                            Positive